
# Importing the Python Libraries to help with reading and manipulating data
import numpy as np
import pandas as pd
from IPython.display import Image

# Importing libraries to help with data visualization
import matplotlib.pyplot as plt
import seaborn as sns
from matplotlib.ticker import PercentFormatter

# Used for Ignore warnings. When we generate the output, then we can use this ignore warning
import warnings

warnings.filterwarnings("ignore")

# To be used for missing value imputation
from sklearn.impute import SimpleImputer
from sklearn.impute import KNNImputer

# this will help in making the Python code more structured automatically (good coding practice)
!pip install nb-black
%reload_ext nb_black

# Command to tell Python to actually display the graphs
%matplotlib inline

# let's start by installing plotly
!pip install plotly

# importing plotly
import plotly.express as px


# Command to hide the 'already satisfied' warnining from displaying
%pip install keras | grep -v 'already satisfied'

# Constant for making bold text
boldText = "\033[1m"

# Removes the limit for the number of displayed columns
pd.set_option("display.max_columns", None)
# Sets the limit for the number of displayed rows
pd.set_option("display.max_rows", 500)

# to split the data into train and test
from sklearn.model_selection import train_test_split

# to build linear regression_model
from sklearn.linear_model import LinearRegression

# to build Bagging model
from sklearn.ensemble import BaggingClassifier, RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier

# to build Boosting model
from sklearn.ensemble import AdaBoostClassifier, GradientBoostingClassifier
from xgboost import XGBClassifier
from sklearn.ensemble import StackingClassifier

from sklearn.experimental import enable_iterative_imputer
from sklearn.impute import IterativeImputer

# to check model performance
from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score

pd.set_option("mode.chained_assignment", None)

# To build model for prediction
from sklearn.linear_model import LogisticRegression

# To get diferent metric scores
# To tune different models
from sklearn.model_selection import GridSearchCV

from sklearn.tree import DecisionTreeClassifier
from sklearn import tree

from sklearn import metrics
from sklearn.metrics import (
    f1_score,
    accuracy_score,
    recall_score,
    precision_score,
    confusion_matrix,
    roc_auc_score,
    plot_confusion_matrix,
    precision_recall_curve,
    roc_curve,
    make_scorer,
)

# To help with model building
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import (
    AdaBoostClassifier,
    GradientBoostingClassifier,
    RandomForestClassifier,
    BaggingClassifier,
)
from xgboost import XGBClassifier

# To get different metric scores, and split data
from sklearn import metrics
from sklearn.model_selection import train_test_split, StratifiedKFold, cross_val_score
from sklearn.metrics import (
    f1_score,
    accuracy_score,
    recall_score,
    precision_score,
    confusion_matrix,
    roc_auc_score,
    plot_confusion_matrix,
)

# To be used for data scaling and one hot encoding
from sklearn.preprocessing import StandardScaler, MinMaxScaler, OneHotEncoder

# To be used for tuning the model
from sklearn.model_selection import GridSearchCV, RandomizedSearchCV

# To be used for creating pipelines and personalizing them
from imblearn.over_sampling import SMOTE
from imblearn.pipeline import Pipeline as imbpipeline
from sklearn.pipeline import Pipeline

from sklearn.compose import ColumnTransformer

# To oversample and undersample data
from imblearn.over_sampling import SMOTE
from imblearn.under_sampling import RandomUnderSampler

# Install library using
# In jupyter notebook
# !pip install shap

# or
# In anaconda command prompt
# conda install -c conda-forge shap - in conda prompt
import shap

Requirement already satisfied: nb-black in c:\users\cpaul\anaconda3\lib\site-packages (1.0.7)
Requirement already satisfied: ipython in c:\users\cpaul\anaconda3\lib\site-packages (from nb-black) (7.29.0)
Requirement already satisfied: black>='19.3' in c:\users\cpaul\anaconda3\lib\site-packages (from nb-black) (19.10b0)
Requirement already satisfied: regex in c:\users\cpaul\anaconda3\lib\site-packages (from black>='19.3'->nb-black) (2021.8.3)
Requirement already satisfied: pathspec<1,>=0.6 in c:\users\cpaul\anaconda3\lib\site-packages (from black>='19.3'->nb-black) (0.7.0)
Requirement already satisfied: typed-ast>=1.4.0 in c:\users\cpaul\anaconda3\lib\site-packages (from black>='19.3'->nb-black) (1.4.3)
Requirement already satisfied: attrs>=18.1.0 in c:\users\cpaul\anaconda3\lib\site-packages (from black>='19.3'->nb-black) (21.2.0)
Requirement already satisfied: appdirs in c:\users\cpaul\anaconda3\lib\site-packages (from black>='19.3'->nb-black) (1.4.4)
Requirement already satisfied: click>=6.5 in c:\users\cpaul\anaconda3\lib\site-packages (from black>='19.3'->nb-black) (8.0.3)
Requirement already satisfied: toml>=0.9.4 in c:\users\cpaul\anaconda3\lib\site-packages (from black>='19.3'->nb-black) (0.10.2)
Requirement already satisfied: colorama in c:\users\cpaul\anaconda3\lib\site-packages (from click>=6.5->black>='19.3'->nb-black) (0.4.4)
Requirement already satisfied: decorator in c:\users\cpaul\anaconda3\lib\site-packages (from ipython->nb-black) (5.1.0)
Requirement already satisfied: jedi>=0.16 in c:\users\cpaul\anaconda3\lib\site-packages (from ipython->nb-black) (0.18.0)
Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in c:\users\cpaul\anaconda3\lib\site-packages (from ipython->nb-black) (3.0.20)
Requirement already satisfied: pickleshare in c:\users\cpaul\anaconda3\lib\site-packages (from ipython->nb-black) (0.7.5)
Requirement already satisfied: pygments in c:\users\cpaul\anaconda3\lib\site-packages (from ipython->nb-black) (2.10.0)
Requirement already satisfied: traitlets>=4.2 in c:\users\cpaul\anaconda3\lib\site-packages (from ipython->nb-black) (5.1.0)
Requirement already satisfied: backcall in c:\users\cpaul\anaconda3\lib\site-packages (from ipython->nb-black) (0.2.0)
Requirement already satisfied: setuptools>=18.5 in c:\users\cpaul\anaconda3\lib\site-packages (from ipython->nb-black) (58.0.4)
Requirement already satisfied: matplotlib-inline in c:\users\cpaul\anaconda3\lib\site-packages (from ipython->nb-black) (0.1.2)
Requirement already satisfied: parso<0.9.0,>=0.8.0 in c:\users\cpaul\anaconda3\lib\site-packages (from jedi>=0.16->ipython->nb-black) (0.8.2)
Requirement already satisfied: wcwidth in c:\users\cpaul\anaconda3\lib\site-packages (from prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0->ipython->nb-black) (0.2.5)
Requirement already satisfied: plotly in c:\users\cpaul\anaconda3\lib\site-packages (5.7.0)
Requirement already satisfied: tenacity>=6.2.0 in c:\users\cpaul\anaconda3\lib\site-packages (from plotly) (8.0.1)
Requirement already satisfied: six in c:\users\cpaul\anaconda3\lib\site-packages (from plotly) (1.16.0)
Note: you may need to restart the kernel to use updated packages.


# Loading Used Cars Dataset
df = pd.read_csv("../Dataset/BankChurners.csv")

# same random results every time
np.random.seed(1)
df.sample(n=10)

# To copy the data to another object
custData = df.copy()


df.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 10127 entries, 0 to 10126
Data columns (total 21 columns):
 #   Column                    Non-Null Count  Dtype  
---  ------                    --------------  -----  
 0   CLIENTNUM                 10127 non-null  int64  
 1   Attrition_Flag            10127 non-null  object 
 2   Customer_Age              10127 non-null  int64  
 3   Gender                    10127 non-null  object 
 4   Dependent_count           10127 non-null  int64  
 5   Education_Level           8608 non-null   object 
 6   Marital_Status            9378 non-null   object 
 7   Income_Category           10127 non-null  object 
 8   Card_Category             10127 non-null  object 
 9   Months_on_book            10127 non-null  int64  
 10  Total_Relationship_Count  10127 non-null  int64  
 11  Months_Inactive_12_mon    10127 non-null  int64  
 12  Contacts_Count_12_mon     10127 non-null  int64  
 13  Credit_Limit              10127 non-null  float64
 14  Total_Revolving_Bal       10127 non-null  int64  
 15  Avg_Open_To_Buy           10127 non-null  float64
 16  Total_Amt_Chng_Q4_Q1      10127 non-null  float64
 17  Total_Trans_Amt           10127 non-null  int64  
 18  Total_Trans_Ct            10127 non-null  int64  
 19  Total_Ct_Chng_Q4_Q1       10127 non-null  float64
 20  Avg_Utilization_Ratio     10127 non-null  float64
dtypes: float64(5), int64(10), object(6)
memory usage: 1.6+ MB


# Command to understand the total number of data collected
print(
    f"- There are {df.shape[0]} row samples and {df.shape[1]} attributes of the customer information collected in this dataset."
)

- There are 10127 row samples and 21 attributes of the customer information collected in this dataset.


df.head(5)  # Displaying the fist 10 rows of the Dataset


df.tail(5)  # Displaying the last 10 rows of the Dataset


df.describe(include="all").T


# creating histograms
df.hist(figsize=(14, 14))
plt.show()


df.nunique()

CLIENTNUM                   10127
Attrition_Flag                  2
Customer_Age                   45
Gender                          2
Dependent_count                 6
Education_Level                 6
Marital_Status                  3
Income_Category                 6
Card_Category                   4
Months_on_book                 44
Total_Relationship_Count        6
Months_Inactive_12_mon          7
Contacts_Count_12_mon           7
Credit_Limit                 6205
Total_Revolving_Bal          1974
Avg_Open_To_Buy              6813
Total_Amt_Chng_Q4_Q1         1158
Total_Trans_Amt              5033
Total_Trans_Ct                126
Total_Ct_Chng_Q4_Q1           830
Avg_Utilization_Ratio         964
dtype: int64


# Dropping the 'ID' columns since its not required
df.drop(["CLIENTNUM"], axis=1, inplace=True)


# Checking for duplicated rows in the dataset
duplicateSum = df.duplicated().sum()
print("**Inferences:**")
if duplicateSum > 0:
    print(f"- There are {str(duplicateSum)} duplicated row(s) in the dataset")
    # Removing the duplicated rows in the dataset
    df.drop_duplicates(inplace=True)
    print(
        f"- There are {str(df.duplicated().sum())} duplicated row(s) in the dataset post cleaning"
    )
    df.duplicated().sum()
    # resetting the index of data frame since some rows will be removed
    df.reset_index(drop=True, inplace=True)
else:
    print("- There are no duplicated row(s) in the dataset")

**Inferences:**
- There are no duplicated row(s) in the dataset


df.isnull().sum()

Attrition_Flag                 0
Customer_Age                   0
Gender                         0
Dependent_count                0
Education_Level             1519
Marital_Status               749
Income_Category                0
Card_Category                  0
Months_on_book                 0
Total_Relationship_Count       0
Months_Inactive_12_mon         0
Contacts_Count_12_mon          0
Credit_Limit                   0
Total_Revolving_Bal            0
Avg_Open_To_Buy                0
Total_Amt_Chng_Q4_Q1           0
Total_Trans_Amt                0
Total_Trans_Ct                 0
Total_Ct_Chng_Q4_Q1            0
Avg_Utilization_Ratio          0
dtype: int64


# printing the number of occurrences of each unique value in each categorical column
num_to_display = 10
for column in df.describe(include="all").columns:
    val_counts = df[column].value_counts(
        dropna=False
    )  # Kept dropNA to False to see the NA value count as well
    print("Unique values in", column, "are :")
    print(val_counts.iloc[:num_to_display])
    if len(val_counts) > num_to_display:
        print(f"Only displaying first {num_to_display} of {len(val_counts)} values.")

    print("-" * 50)
    print(" ")

Unique values in Attrition_Flag are :
Existing Customer    8500
Attrited Customer    1627
Name: Attrition_Flag, dtype: int64
--------------------------------------------------
 
Unique values in Customer_Age are :
44    500
49    495
46    490
45    486
47    479
43    473
48    472
50    452
42    426
51    398
Name: Customer_Age, dtype: int64
Only displaying first 10 of 45 values.
--------------------------------------------------
 
Unique values in Gender are :
F    5358
M    4769
Name: Gender, dtype: int64
--------------------------------------------------
 
Unique values in Dependent_count are :
3    2732
2    2655
1    1838
4    1574
0     904
5     424
Name: Dependent_count, dtype: int64
--------------------------------------------------
 
Unique values in Education_Level are :
Graduate         3128
High School      2013
NaN              1519
Uneducated       1487
College          1013
Post-Graduate     516
Doctorate         451
Name: Education_Level, dtype: int64
--------------------------------------------------
 
Unique values in Marital_Status are :
Married     4687
Single      3943
NaN          749
Divorced     748
Name: Marital_Status, dtype: int64
--------------------------------------------------
 
Unique values in Income_Category are :
Less than $40K    3561
$40K - $60K       1790
$80K - $120K      1535
$60K - $80K       1402
abc               1112
$120K +            727
Name: Income_Category, dtype: int64
--------------------------------------------------
 
Unique values in Card_Category are :
Blue        9436
Silver       555
Gold         116
Platinum      20
Name: Card_Category, dtype: int64
--------------------------------------------------
 
Unique values in Months_on_book are :
36    2463
37     358
34     353
38     347
39     341
40     333
31     318
35     317
33     305
30     300
Name: Months_on_book, dtype: int64
Only displaying first 10 of 44 values.
--------------------------------------------------
 
Unique values in Total_Relationship_Count are :
3    2305
4    1912
5    1891
6    1866
2    1243
1     910
Name: Total_Relationship_Count, dtype: int64
--------------------------------------------------
 
Unique values in Months_Inactive_12_mon are :
3    3846
2    3282
1    2233
4     435
5     178
6     124
0      29
Name: Months_Inactive_12_mon, dtype: int64
--------------------------------------------------
 
Unique values in Contacts_Count_12_mon are :
3    3380
2    3227
1    1499
4    1392
0     399
5     176
6      54
Name: Contacts_Count_12_mon, dtype: int64
--------------------------------------------------
 
Unique values in Credit_Limit are :
34516.0    508
1438.3     507
9959.0      18
15987.0     18
23981.0     12
2490.0      11
6224.0      11
3735.0      11
7469.0      10
2069.0       8
Name: Credit_Limit, dtype: int64
Only displaying first 10 of 6205 values.
--------------------------------------------------
 
Unique values in Total_Revolving_Bal are :
0       2470
2517     508
1965      12
1480      12
1434      11
1664      11
1720      11
1590      10
1542      10
1528      10
Name: Total_Revolving_Bal, dtype: int64
Only displaying first 10 of 1974 values.
--------------------------------------------------
 
Unique values in Avg_Open_To_Buy are :
1438.3     324
34516.0     98
31999.0     26
787.0        8
701.0        7
713.0        7
953.0        7
463.0        7
990.0        6
788.0        6
Name: Avg_Open_To_Buy, dtype: int64
Only displaying first 10 of 6813 values.
--------------------------------------------------
 
Unique values in Total_Amt_Chng_Q4_Q1 are :
0.791    36
0.712    34
0.743    34
0.718    33
0.735    33
0.744    32
0.699    32
0.722    32
0.731    31
0.631    31
Name: Total_Amt_Chng_Q4_Q1, dtype: int64
Only displaying first 10 of 1158 values.
--------------------------------------------------
 
Unique values in Total_Trans_Amt are :
4253    11
4509    11
4518    10
2229    10
4220     9
4869     9
4037     9
4313     9
4498     9
4042     9
Name: Total_Trans_Amt, dtype: int64
Only displaying first 10 of 5033 values.
--------------------------------------------------
 
Unique values in Total_Trans_Ct are :
81    208
71    203
75    203
69    202
82    202
76    198
77    197
70    193
74    190
78    190
Name: Total_Trans_Ct, dtype: int64
Only displaying first 10 of 126 values.
--------------------------------------------------
 
Unique values in Total_Ct_Chng_Q4_Q1 are :
0.667    171
1.000    166
0.500    161
0.750    156
0.600    113
0.800    101
0.714     92
0.833     85
0.778     69
0.625     63
Name: Total_Ct_Chng_Q4_Q1, dtype: int64
Only displaying first 10 of 830 values.
--------------------------------------------------
 
Unique values in Avg_Utilization_Ratio are :
0.000    2470
0.073      44
0.057      33
0.048      32
0.060      30
0.061      29
0.045      29
0.059      28
0.069      28
0.053      27
Name: Avg_Utilization_Ratio, dtype: int64
Only displaying first 10 of 964 values.
--------------------------------------------------


df["Attrition_Flag"] = df["Attrition_Flag"].astype("category")
df["Gender"] = df["Gender"].astype("category")
df["Dependent_count"] = df["Dependent_count"].astype("category")
df["Education_Level"] = df["Education_Level"].astype("category")
df["Marital_Status"] = df["Marital_Status"].astype("category")
df["Income_Category"] = df["Income_Category"].astype("category")
df["Card_Category"] = df["Card_Category"].astype("category")
df["Total_Relationship_Count"] = df["Total_Relationship_Count"].astype("category")
df["Months_Inactive_12_mon"] = df["Months_Inactive_12_mon"].astype("category")
df["Contacts_Count_12_mon"] = df["Contacts_Count_12_mon"].astype("category")


# The incorrect type in Income ccategory is replaced with NaN and will be addressed as part of missing values
df.Income_Category = df.Income_Category.replace("abc", np.nan)


# Replacing the text values of the Target Variable with 0 (Existing) & 1 (Attrition)
att_flag = {"Existing Customer": 0, "Attrited Customer": 1}
df["Attrition_Flag"] = df["Attrition_Flag"].map(att_flag)

#df.Attrition_Flag = df.Attrition_Flag.replace("Existing Customer", 0)
#df.Attrition_Flag = df.Attrition_Flag.replace("Attrited Customer", 1)


# Defining bins for splitting the age to groups and creating a new column to review the relationship
bins = [20, 30, 40, 50, 60, 70, 80]
labels = [
    "Less_than_30",
    "Less_than_40",
    "Less_than_50",
    "Less_than_60",
    "Less_than_70",
    "Less_than_80",
]
df["AgeGroup"] = pd.cut(df["Customer_Age"], bins=bins, labels=labels, right=False)
df["AgeGroup"] = df["AgeGroup"].astype("category")

df["AgeGroup"].value_counts(dropna=False)

Less_than_50    4561
Less_than_60    2998
Less_than_40    1841
Less_than_70     530
Less_than_30     195
Less_than_80       2
Name: AgeGroup, dtype: int64


# Defining bins for splitting the relationship years to groups and creating a new column to review the relationship
bins = [0, 12, 24, 36, 48, 60, 72]
labels = [
    "Between_0-1_Year",
    "Between_1-2_Year",
    "Between_2-3_Year",
    "Between_3-4_Year",
    "Between_4-5_Year",
    "Between_5-6_Year",
]
df["Months_on_book_Grp"] = pd.cut(
    df["Months_on_book"], bins=bins, labels=labels, right=False
)
df["Months_on_book_Grp"] = df["Months_on_book_Grp"].astype("category")

df["Months_on_book_Grp"].value_counts(dropna=False)

Between_3-4_Year    5508
Between_2-3_Year    3115
Between_4-5_Year     817
Between_1-2_Year     687
Between_0-1_Year       0
Between_5-6_Year       0
Name: Months_on_book_Grp, dtype: int64


# Defining bins for splitting the credit limits to groups and creating a new column to review the relationship
bins = [0, 5000, 10000, 15000, 20000, 25000, 40000]
labels = [
    "<5K",
    "Between_5K-10K",
    "Between_10K-15K",
    "Between_15K-20K",
    "Between_20K-25K",
    ">25K",
]
df["Credit_Limit_Grp"] = pd.cut(
    df["Credit_Limit"], bins=bins, labels=labels, right=False
)
df["Credit_Limit_Grp"] = df["Credit_Limit_Grp"].astype("category")

df["Credit_Limit_Grp"].value_counts(dropna=False)

<5K                5358
Between_5K-10K     2015
Between_10K-15K     941
>25K                892
Between_15K-20K     549
Between_20K-25K     372
Name: Credit_Limit_Grp, dtype: int64


# Observing the data dictionery after the changes
df.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 10127 entries, 0 to 10126
Data columns (total 23 columns):
 #   Column                    Non-Null Count  Dtype   
---  ------                    --------------  -----   
 0   Attrition_Flag            10127 non-null  category
 1   Customer_Age              10127 non-null  int64   
 2   Gender                    10127 non-null  category
 3   Dependent_count           10127 non-null  category
 4   Education_Level           8608 non-null   category
 5   Marital_Status            9378 non-null   category
 6   Income_Category           9015 non-null   category
 7   Card_Category             10127 non-null  category
 8   Months_on_book            10127 non-null  int64   
 9   Total_Relationship_Count  10127 non-null  category
 10  Months_Inactive_12_mon    10127 non-null  category
 11  Contacts_Count_12_mon     10127 non-null  category
 12  Credit_Limit              10127 non-null  float64 
 13  Total_Revolving_Bal       10127 non-null  int64   
 14  Avg_Open_To_Buy           10127 non-null  float64 
 15  Total_Amt_Chng_Q4_Q1      10127 non-null  float64 
 16  Total_Trans_Amt           10127 non-null  int64   
 17  Total_Trans_Ct            10127 non-null  int64   
 18  Total_Ct_Chng_Q4_Q1       10127 non-null  float64 
 19  Avg_Utilization_Ratio     10127 non-null  float64 
 20  AgeGroup                  10127 non-null  category
 21  Months_on_book_Grp        10127 non-null  category
 22  Credit_Limit_Grp          10127 non-null  category
dtypes: category(13), float64(5), int64(5)
memory usage: 922.6 KB


# Command to understand the total number of data collected
print(
    f"- There are {df.shape[0]} row samples and {df.shape[1]} attributes of the customer information collected in this dataset."
)

- There are 10127 row samples and 23 attributes of the customer information collected in this dataset.


# Identofying the category columns
category_columnNames = df.describe(include=["category"]).columns
category_columnNames

Index(['Attrition_Flag', 'Gender', 'Dependent_count', 'Education_Level',
       'Marital_Status', 'Income_Category', 'Card_Category',
       'Total_Relationship_Count', 'Months_Inactive_12_mon',
       'Contacts_Count_12_mon', 'AgeGroup', 'Months_on_book_Grp',
       'Credit_Limit_Grp'],
      dtype='object')


# Identifying the numerical columns
number_columnNames = (
    df.describe(include=["int64"]).columns.tolist()
    + df.describe(include=["float64"]).columns.tolist()
)
number_columnNames

['Customer_Age',
 'Months_on_book',
 'Total_Revolving_Bal',
 'Total_Trans_Amt',
 'Total_Trans_Ct',
 'Credit_Limit',
 'Avg_Open_To_Buy',
 'Total_Amt_Chng_Q4_Q1',
 'Total_Ct_Chng_Q4_Q1',
 'Avg_Utilization_Ratio']


df.describe(include="category").T


df.describe().T


# function to create labeled barplots


def labeled_barplot(data, feature, perc=False, n=None, hueCol=None):
    """
    Barplot with percentage at the top

    data: dataframe
    feature: dataframe column
    perc: whether to display percentages instead of count (default is False)
    n: displays the top n category levels (default is None, i.e., display all levels)
    """

    total = len(data[feature])  # length of the column
    count = data[feature].nunique()
    if n is None:
        plt.figure(figsize=(count + 1, 5))
    else:
        plt.figure(figsize=(n + 1, 7))

    plt.xticks(rotation=90, fontsize=15)

    ax = sns.countplot(
        data=data,
        x=feature,
        palette="Paired",
        hue=hueCol,
        order=data[feature].value_counts().index[:n].sort_values(),
    )

    for p in ax.patches:
        if perc == True:
            label = "{:.1f}%".format(
                100 * p.get_height() / total
            )  # percentage of each class of the category
        else:
            label = p.get_height()  # count of each level of the category

        x = p.get_x() + p.get_width() / 2  # width of the plot
        y = p.get_height()  # height of the plot

        ax.annotate(
            label,
            (x, y),
            ha="center",
            va="center",
            size=12,
            xytext=(0, 5),
            textcoords="offset points",
        )
        # annotate the percentage

    plt.show()  # show the plot


def stacked_barplot(data, predictor, target):
    """
    Print the category counts and plot a stacked bar chart

    data: dataframe
    predictor: independent variable
    target: target variable
    """
    count = data[predictor].nunique()
    sorter = data[target].value_counts().index[-1]
    tab1 = pd.crosstab(data[predictor], data[target], margins=True,).sort_values(
        by=sorter, ascending=False
    )
    print("-" * 30, " Volume ", "-" * 30)
    print(tab1)

    tab1 = pd.crosstab(
        data[predictor], data[target], margins=True, normalize="index"
    ).sort_values(by=sorter, ascending=False)
    print("-" * 30, " Percentage % ", "-" * 30)
    print(tab1)
    print("-" * 120)
    tab = pd.crosstab(data[predictor], data[target], normalize="index").sort_values(
        by=sorter, ascending=False
    )
    tab.plot(kind="bar", stacked=True, figsize=(count + 5, 5))
    plt.legend(
        loc="lower left", frameon=False,
    )
    plt.legend(loc="upper left", bbox_to_anchor=(1, 1))
    plt.show()


# Creating a common function to draw a Boxplot & a Histogram for each of the analysis
def histogram_boxplot(data, feature, figsize=(15, 7), kde=True, bins=None):
    f2, (ax_box2, ax_hist2) = plt.subplots(
        nrows=2,  # Number of rows of the subplot grid= 2
        sharex=True,  # x-axis will be shared among all subplots
        gridspec_kw={"height_ratios": (0.25, 0.75)},
        figsize=figsize,
    )  # creating the 2 subplots

    sns.boxplot(
        data=data, x=feature, ax=ax_box2, showmeans=True, color="violet"
    )  # boxplot will be created and a star will indicate the mean value of the column

    if bins:
        sns.histplot(
            data=data, x=feature, kde=kde, ax=ax_hist2, bins=bins, palette="winter"
        )
    else:
        sns.histplot(data=data, x=feature, kde=kde, ax=ax_hist2)  # For histogram

    ax_hist2.axvline(
        data[feature].mean(), color="green", linestyle="--"
    )  # Add mean to the histogram

    ax_hist2.axvline(
        data[feature].median(), color="black", linestyle="-"
    )  # Add median to the histogram


# functions to treat outliers by flooring and capping


def treat_outliers(df, col, lower=0.25, upper=0.75, mul=1.5):
    """
    Treats outliers in a variable

    df: dataframe
    col: dataframe column
    """
    Q1 = df[col].quantile(lower)  # 25th quantile
    Q3 = df[col].quantile(upper)  # 75th quantile
    IQR = Q3 - Q1
    Lower_Whisker = Q1 - (mul * IQR)
    Upper_Whisker = Q3 + (mul * IQR)

    # all the values smaller than Lower_Whisker will be assigned the value of Lower_Whisker
    # all the values greater than Upper_Whisker will be assigned the value of Upper_Whisker
    df[col] = np.clip(df[col], Lower_Whisker, Upper_Whisker)

    return df


def treat_outliers_all(df, col_list, lower=0.25, upper=0.75, mul=1.5):
    """
    Treat outliers in a list of variables

    df: dataframe
    col_list: list of dataframe columns
    """
    for c in col_list:
        df = treat_outliers(df, c, lower, upper, mul)

    return df


# Summary of data
df.describe(include="all").T


# printing the number of occurrences of each unique value in each categorical column

num_to_display = 15
for column in category_columnNames:
    val_counts = df[column].value_counts(
        dropna=False
    )  # Kept dropNA to False to see the NA value count as well
    #val_countsP = df[column].value_counts(dropna=False, normalize=True)
    print("Unique values in", column, "are :")
    print(val_counts.iloc[:num_to_display])
    #print(val_countsP.iloc[:num_to_display])
    if len(val_counts) > num_to_display:
        print(f"Only displaying first {num_to_display} of {len(val_counts)} values.")

    labeled_barplot(df, column, perc=True, n=5)
    plt.tight_layout()

    print("-" * 50)
    print(" ")

Unique values in Attrition_Flag are :
0    8500
1    1627
Name: Attrition_Flag, dtype: int64

--------------------------------------------------
 
Unique values in Gender are :
F    5358
M    4769
Name: Gender, dtype: int64

<Figure size 432x288 with 0 Axes>

--------------------------------------------------
 
Unique values in Dependent_count are :
3    2732
2    2655
1    1838
4    1574
0     904
5     424
Name: Dependent_count, dtype: int64

<Figure size 432x288 with 0 Axes>

--------------------------------------------------
 
Unique values in Education_Level are :
Graduate         3128
High School      2013
NaN              1519
Uneducated       1487
College          1013
Post-Graduate     516
Doctorate         451
Name: Education_Level, dtype: int64

<Figure size 432x288 with 0 Axes>

--------------------------------------------------
 
Unique values in Marital_Status are :
Married     4687
Single      3943
NaN          749
Divorced     748
Name: Marital_Status, dtype: int64

<Figure size 432x288 with 0 Axes>

--------------------------------------------------
 
Unique values in Income_Category are :
Less than $40K    3561
$40K - $60K       1790
$80K - $120K      1535
$60K - $80K       1402
NaN               1112
$120K +            727
Name: Income_Category, dtype: int64


# creating histograms
df.hist(figsize=(14, 14))
plt.show()


# Summary of numeric data
df.describe().T


histogram_boxplot(df, "Customer_Age")


histogram_boxplot(df, "Months_on_book")


df[df.Months_on_book > 50]["Months_on_book"].describe()

count    418.000000
mean      53.535885
std        1.838805
min       51.000000
25%       52.000000
50%       53.000000
75%       55.000000
max       56.000000
Name: Months_on_book, dtype: float64


histogram_boxplot(df, "Credit_Limit")


df[(df['Credit_Limit'] > 20000)]["Income_Category"].value_counts()

$80K - $120K      520
$120K +           344
$60K - $80K       233
$40K - $60K        27
Less than $40K      0
Name: Income_Category, dtype: int64


# Finding the median values of the Credit Limit with respective to the Income type
df.groupby(["Income_Category"])[["Credit_Limit"]].median()


# Finding the median values of the Credit Limit with respective to the Card Category
df.groupby(["Card_Category"])[["Credit_Limit"]].median()


df.groupby(["Income_Category", "Card_Category"])[["Credit_Limit"]].median()


df['Credit_Limit'] = np.where(
    ((df['Credit_Limit'] > 20000) & (df['Income_Category'] == "$40K - $60K")) , 3682, df['Credit_Limit'])

df['Credit_Limit'] = np.where(
    ((df['Credit_Limit'] > 20000) & (df['Income_Category'] == "$60K - $80K")) , 7660, df['Credit_Limit'])

df['Credit_Limit'] = np.where(
    ((df['Credit_Limit'] > 20000) & (df['Income_Category'] == "$80K - $120K")) , 12830, df['Credit_Limit'])

df['Credit_Limit'] = np.where(
    ((df['Credit_Limit'] > 20000) & (df['Income_Category'] == "$120K +")) , 18442, df['Credit_Limit'])

df['Credit_Limit'] = np.where(
    ((df['Credit_Limit'] > 20000) & (df['Income_Category'].isna())) , df['Credit_Limit'].mean(), df['Credit_Limit'])


df[(df["Credit_Limit"] > 20000)].head()


histogram_boxplot(df, "Credit_Limit")


histogram_boxplot(df, "Total_Revolving_Bal")


histogram_boxplot(df, "Avg_Open_To_Buy")


df[(df["Avg_Open_To_Buy"] > 20000)]["Income_Category"].value_counts()

$80K - $120K      486
$120K +           328
$60K - $80K       205
$40K - $60K        22
Less than $40K      0
Name: Income_Category, dtype: int64


# Finding the median values of the Avg_Open_To_Buy with respective to the Income type
df.groupby(["Income_Category"])[["Avg_Open_To_Buy"]].median()


df["Avg_Open_To_Buy"] = np.where(
    ((df["Avg_Open_To_Buy"] > 20000) & (df["Income_Category"] == "$40K - $60K")),
    2580.5,
    df["Avg_Open_To_Buy"],
)

df["Avg_Open_To_Buy"] = np.where(
    ((df["Avg_Open_To_Buy"] > 20000) & (df["Income_Category"] == "$60K - $80K")),
    6418.5,
    df["Avg_Open_To_Buy"],
)

df["Avg_Open_To_Buy"] = np.where(
    ((df["Avg_Open_To_Buy"] > 20000) & (df["Income_Category"] == "$80K - $120K")),
    11606,
    df["Avg_Open_To_Buy"],
)

df["Avg_Open_To_Buy"] = np.where(
    ((df["Avg_Open_To_Buy"] > 20000) & (df["Income_Category"] == "$120K +")),
    17117,
    df["Avg_Open_To_Buy"],
)

df["Avg_Open_To_Buy"] = np.where(
    ((df["Avg_Open_To_Buy"] > 20000) & (df["Income_Category"].isna())),
    df["Avg_Open_To_Buy"].mean(),
    df["Avg_Open_To_Buy"],
)


histogram_boxplot(df, "Avg_Open_To_Buy")


histogram_boxplot(df, "Total_Amt_Chng_Q4_Q1")


df = treat_outliers(df, "Total_Amt_Chng_Q4_Q1", 0.25, 0.75, 1.5)
histogram_boxplot(df, "Total_Amt_Chng_Q4_Q1")


histogram_boxplot(df, "Total_Ct_Chng_Q4_Q1")


df = treat_outliers(df, "Total_Ct_Chng_Q4_Q1", 0.25, 0.75, 1.5)
histogram_boxplot(df, "Total_Ct_Chng_Q4_Q1")


histogram_boxplot(df, "Total_Trans_Amt")


df.groupby(["Income_Category"])[["Credit_Limit"]].median()


df.groupby(["Card_Category"])[["Credit_Limit"]].median()


df.groupby(["Card_Category", "Income_Category"])[["Credit_Limit"]].median()


df.groupby(["Card_Category", "Income_Category"])[["Total_Trans_Amt"]].median()


histogram_boxplot(df, "Total_Trans_Ct")


histogram_boxplot(df, "Avg_Utilization_Ratio")


df.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 10127 entries, 0 to 10126
Data columns (total 23 columns):
 #   Column                    Non-Null Count  Dtype   
---  ------                    --------------  -----   
 0   Attrition_Flag            10127 non-null  category
 1   Customer_Age              10127 non-null  int64   
 2   Gender                    10127 non-null  category
 3   Dependent_count           10127 non-null  category
 4   Education_Level           8608 non-null   category
 5   Marital_Status            9378 non-null   category
 6   Income_Category           9015 non-null   category
 7   Card_Category             10127 non-null  category
 8   Months_on_book            10127 non-null  int64   
 9   Total_Relationship_Count  10127 non-null  category
 10  Months_Inactive_12_mon    10127 non-null  category
 11  Contacts_Count_12_mon     10127 non-null  category
 12  Credit_Limit              10127 non-null  float64 
 13  Total_Revolving_Bal       10127 non-null  int64   
 14  Avg_Open_To_Buy           10127 non-null  float64 
 15  Total_Amt_Chng_Q4_Q1      10127 non-null  float64 
 16  Total_Trans_Amt           10127 non-null  int64   
 17  Total_Trans_Ct            10127 non-null  int64   
 18  Total_Ct_Chng_Q4_Q1       10127 non-null  float64 
 19  Avg_Utilization_Ratio     10127 non-null  float64 
 20  AgeGroup                  10127 non-null  category
 21  Months_on_book_Grp        10127 non-null  category
 22  Credit_Limit_Grp          10127 non-null  category
dtypes: category(13), float64(5), int64(5)
memory usage: 922.6 KB


# Data Secription of Categorical variables
df.describe(include="category").T


# Data Secription of Categorical variables
df.describe().T


# Creating a function to display the values of the attributes against the chart
for i, cols in zip(range(len(category_columnNames)), category_columnNames):
    count = df[cols].nunique()
    sorter = df["Attrition_Flag"].value_counts(dropna=False).index[-1]
    tab1 = pd.crosstab(df[cols], df["Attrition_Flag"], margins=True,).sort_values(
        by=sorter, ascending=False
    )
    print("-" * 30, " Volume ", "-" * 30)
    print(tab1)

    tab1 = pd.crosstab(
        df[cols], df["Attrition_Flag"], margins=True, normalize="index"
    ).sort_values(by=sorter, ascending=False)
    print("-" * 30, " Percentage % ", "-" * 30)
    print(tab1)
    print("-" * 120)
    
    labeled_barplot(df, cols, perc=True, n=10, hueCol="Attrition_Flag")
    plt.tight_layout()

------------------------------  Volume  ------------------------------
Attrition_Flag     1     0    All
Attrition_Flag                   
1               1627     0   1627
All             1627  8500  10127
0                  0  8500   8500
------------------------------  Percentage %  ------------------------------
Attrition_Flag        1        0
Attrition_Flag                  
1               1.00000  0.00000
All             0.16066  0.83934
0               0.00000  1.00000
------------------------------------------------------------------------------------------------------------------------

------------------------------  Volume  ------------------------------
Attrition_Flag     1     0    All
Gender                           
All             1627  8500  10127
F                930  4428   5358
M                697  4072   4769
------------------------------  Percentage %  ------------------------------
Attrition_Flag         1         0
Gender                            
F               0.173572  0.826428
All             0.160660  0.839340
M               0.146152  0.853848
------------------------------------------------------------------------------------------------------------------------

<Figure size 432x288 with 0 Axes>

------------------------------  Volume  ------------------------------
Attrition_Flag      1     0    All
Dependent_count                   
All              1627  8500  10127
3                 482  2250   2732
2                 417  2238   2655
1                 269  1569   1838
4                 260  1314   1574
0                 135   769    904
5                  64   360    424
------------------------------  Percentage %  ------------------------------
Attrition_Flag          1         0
Dependent_count                    
3                0.176428  0.823572
4                0.165184  0.834816
All              0.160660  0.839340
2                0.157062  0.842938
5                0.150943  0.849057
0                0.149336  0.850664
1                0.146355  0.853645
------------------------------------------------------------------------------------------------------------------------

<Figure size 432x288 with 0 Axes>

------------------------------  Volume  ------------------------------
Attrition_Flag      1     0   All
Education_Level                  
All              1371  7237  8608
Graduate          487  2641  3128
High School       306  1707  2013
Uneducated        237  1250  1487
College           154   859  1013
Doctorate          95   356   451
Post-Graduate      92   424   516
------------------------------  Percentage %  ------------------------------
Attrition_Flag          1         0
Education_Level                    
Doctorate        0.210643  0.789357
Post-Graduate    0.178295  0.821705
Uneducated       0.159381  0.840619
All              0.159270  0.840730
Graduate         0.155691  0.844309
College          0.152024  0.847976
High School      0.152012  0.847988
------------------------------------------------------------------------------------------------------------------------

<Figure size 432x288 with 0 Axes>

------------------------------  Volume  ------------------------------
Attrition_Flag     1     0   All
Marital_Status                  
All             1498  7880  9378
Married          709  3978  4687
Single           668  3275  3943
Divorced         121   627   748
------------------------------  Percentage %  ------------------------------
Attrition_Flag         1         0
Marital_Status                    
Single          0.169414  0.830586
Divorced        0.161765  0.838235
All             0.159736  0.840264
Married         0.151269  0.848731
------------------------------------------------------------------------------------------------------------------------

<Figure size 432x288 with 0 Axes>

------------------------------  Volume  ------------------------------
Attrition_Flag      1     0   All
Income_Category                  
All              1440  7575  9015
Less than $40K    612  2949  3561
$40K - $60K       271  1519  1790
$80K - $120K      242  1293  1535
$60K - $80K       189  1213  1402
$120K +           126   601   727
------------------------------  Percentage %  ------------------------------
Attrition_Flag          1         0
Income_Category                    
$120K +          0.173315  0.826685
Less than $40K   0.171862  0.828138
All              0.159734  0.840266
$80K - $120K     0.157655  0.842345
$40K - $60K      0.151397  0.848603
$60K - $80K      0.134807  0.865193
------------------------------------------------------------------------------------------------------------------------


plt.figure(figsize=(15, 20))

line_columnnames=['Gender', 'Dependent_count', 'Education_Level',
       'Marital_Status', 'Income_Category', 'Card_Category',
       'Total_Relationship_Count', 'Months_Inactive_12_mon']

for i, variable in enumerate(line_columnnames):
    plt.subplot(5, 2, i + 1)
    sns.lineplot(x=variable, y="Attrition_Flag", data=df)
    plt.tight_layout()
    plt.title(variable)
plt.show()


plt.figure(figsize=(15, 25))

for i, variable in enumerate(number_columnNames):
    plt.subplot(5, 2, i + 1)
    sns.boxplot(df["Attrition_Flag"], df[variable], palette="PuBu", showfliers=False)
    plt.tight_layout()
    plt.title(variable)
plt.show()


plt.figure(figsize=(15, 20))

line_columnnames=['Customer_Age', 'Months_on_book', 'Total_Trans_Ct']

for i, variable in enumerate(line_columnnames):
    plt.subplot(5, 2, i + 1)
    sns.lineplot(x=variable, y="Attrition_Flag", data=df)
    plt.tight_layout()
    plt.title(variable)
plt.show()


# Plotting Heatmap by creating a 2-D Matrix with correlation plots
correlation = df.corr()
plt.figure(figsize=(15, 7))
sns.heatmap(correlation, vmin=-1, vmax=1, annot=True, cmap="Spectral")

<AxesSubplot:>


sns.pairplot(df, corner=True, hue="Attrition_Flag")

<seaborn.axisgrid.PairGrid at 0x1f3409cab20>


tab = pd.crosstab(df["Card_Category"], df["Income_Category"], normalize="index")
tab.plot(kind="bar", stacked=True)
plt.show()


tab = pd.crosstab(df["AgeGroup"], df["Months_on_book_Grp"], normalize="index")
tab.plot(kind="bar", stacked=True)
plt.show()


plt.figure(figsize=(15, 7))
sns.boxplot(x="Total_Trans_Amt", y="Income_Category", data=df, hue="Attrition_Flag")
plt.show()


plt.figure(figsize=(15, 7))
sns.boxplot(x="Dependent_count", y="Total_Trans_Amt", data=df, hue="Attrition_Flag")
plt.show()


g = sns.FacetGrid(
    df, col="Income_Category", hue="Attrition_Flag", col_wrap=4, margin_titles=True
)
g.map(sns.scatterplot, "Credit_Limit", "Customer_Age")
g.add_legend()

<seaborn.axisgrid.FacetGrid at 0x1f33b1146d0>


g = sns.FacetGrid(
    df, col="Income_Category", hue="Attrition_Flag", col_wrap=4, margin_titles=True
)
g.map(sns.scatterplot, "Total_Trans_Amt", "Customer_Age")
g.add_legend()

<seaborn.axisgrid.FacetGrid at 0x1f341ea4a00>


g = sns.FacetGrid(
    df, col="Card_Category", hue="Attrition_Flag", col_wrap=4, margin_titles=True
)
g.map(sns.scatterplot, "Credit_Limit", "Total_Trans_Amt")
g.add_legend()

<seaborn.axisgrid.FacetGrid at 0x1f341edc760>


plt.figure(figsize=(15, 7))
sns.boxplot(x="Card_Category", y="Total_Trans_Amt", data=df, hue="Attrition_Flag")
plt.show()


# defining a function to compute different metrics to check performance of a classification model built using sklearn

def model_performance_classification_sklearn_with_threshold(model, predictors, target, threshold=0.5):
    """
    Function to compute different metrics, based on the threshold specified, to check classification model performance

    model: classifier
    predictors: independent variables
    target: dependent variable
    threshold: threshold for classifying the observation as class 1
    """

    # predicting using the independent variables
    pred_prob = model.predict_proba(predictors)[:, 1] 
    pred_thres = pred_prob > threshold
    pred = np.round(pred_thres)

    #target, pred
    acc = accuracy_score(target, pred)  # to compute Accuracy
    recall = recall_score(target, pred)  # to compute Recall
    precision = precision_score(target, pred)  # to compute Precision
    f1 = f1_score(target, pred)  # to compute F1-score

    # creating a dataframe of metrics
    df_perf = pd.DataFrame(
        {
            "Accuracy": acc,
            "Recall": recall,
            "Precision": precision,
            "F1": f1,
        },
        index=[0],
    )

    return df_perf


# defining a function to plot the confusion_matrix of a classification model built using sklearn
def confusion_matrix_sklearn_with_threshold(model, predictors, target, threshold=0.5):
    """
    To plot the confusion_matrix, based on the threshold specified, with percentages

    model: classifier
    predictors: independent variables
    target: dependent variable
    threshold: threshold for classifying the observation as class 1
    """
    pred_prob = model.predict_proba(predictors)[:, 1]  # Probability answer.
    pred_thres = pred_prob > threshold
    y_pred = np.round(pred_thres)

    cm = confusion_matrix(target, y_pred)
    labels = np.asarray(
        [
            ["{0:0.0f}".format(item) + "\n{0:.2%}".format(item / cm.flatten().sum())]
            for item in cm.flatten()
        ]
    ).reshape(2, 2)

    plt.figure(figsize=(6, 4))
    sns.heatmap(cm, annot=labels, fmt="")
    plt.ylabel("True label")
    plt.xlabel("Predicted label")


df.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 10127 entries, 0 to 10126
Data columns (total 23 columns):
 #   Column                    Non-Null Count  Dtype   
---  ------                    --------------  -----   
 0   Attrition_Flag            10127 non-null  category
 1   Customer_Age              10127 non-null  int64   
 2   Gender                    10127 non-null  category
 3   Dependent_count           10127 non-null  category
 4   Education_Level           8608 non-null   category
 5   Marital_Status            9378 non-null   category
 6   Income_Category           9015 non-null   category
 7   Card_Category             10127 non-null  category
 8   Months_on_book            10127 non-null  int64   
 9   Total_Relationship_Count  10127 non-null  category
 10  Months_Inactive_12_mon    10127 non-null  category
 11  Contacts_Count_12_mon     10127 non-null  category
 12  Credit_Limit              10127 non-null  float64 
 13  Total_Revolving_Bal       10127 non-null  int64   
 14  Avg_Open_To_Buy           10127 non-null  float64 
 15  Total_Amt_Chng_Q4_Q1      10127 non-null  float64 
 16  Total_Trans_Amt           10127 non-null  int64   
 17  Total_Trans_Ct            10127 non-null  int64   
 18  Total_Ct_Chng_Q4_Q1       10127 non-null  float64 
 19  Avg_Utilization_Ratio     10127 non-null  float64 
 20  AgeGroup                  10127 non-null  category
 21  Months_on_book_Grp        10127 non-null  category
 22  Credit_Limit_Grp          10127 non-null  category
dtypes: category(13), float64(5), int64(5)
memory usage: 922.6 KB


X = df.drop(["Attrition_Flag"], axis=1)
y = df["Attrition_Flag"]


# Dropping off the following columns since they will not play a part in determing the model for the customers purchasing the new product
X.drop(["AgeGroup"], axis=1, inplace=True)
X.drop(["Months_on_book_Grp"], axis=1, inplace=True)
X.drop(["Credit_Limit_Grp"], axis=1, inplace=True)
X.drop(["Avg_Utilization_Ratio"], axis=1, inplace=True)
X.drop(["Total_Ct_Chng_Q4_Q1"], axis=1, inplace=True)
X.drop(["Total_Amt_Chng_Q4_Q1"], axis=1, inplace=True)


# Splitting data into training, validation and test sets:
# first we split data into 2 parts, say temporary and test

X_temp, X_test, y_temp, y_test = train_test_split(
    X, y, test_size=0.2, random_state=1, stratify=y
)

# then we split the temporary set into train and validation

X_train, X_val, y_train, y_val = train_test_split(
    X_temp, y_temp, test_size=0.25, random_state=1, stratify=y_temp
)
print(X_train.shape, X_val.shape, X_test.shape)

(6075, 16) (2026, 16) (2026, 16)


# Let's impute the missing values
imp_mode = SimpleImputer(missing_values=np.nan, strategy="most_frequent")
cols_to_impute = ["Education_Level", "Marital_Status", "Income_Category"]

# fit and transform the imputer on train data
X_train[cols_to_impute] = imp_mode.fit_transform(X_train[cols_to_impute])

# Transform on validation and test data
X_val[cols_to_impute] = imp_mode.transform(X_val[cols_to_impute])

# fit and transform the imputer on test data
X_test[cols_to_impute] = imp_mode.transform(X_test[cols_to_impute])


# Checking that no column has missing values in train or test sets
print(X_train.isna().sum())
print("-" * 30)
print(X_test.isna().sum())

Customer_Age                0
Gender                      0
Dependent_count             0
Education_Level             0
Marital_Status              0
Income_Category             0
Card_Category               0
Months_on_book              0
Total_Relationship_Count    0
Months_Inactive_12_mon      0
Contacts_Count_12_mon       0
Credit_Limit                0
Total_Revolving_Bal         0
Avg_Open_To_Buy             0
Total_Trans_Amt             0
Total_Trans_Ct              0
dtype: int64
------------------------------
Customer_Age                0
Gender                      0
Dependent_count             0
Education_Level             0
Marital_Status              0
Income_Category             0
Card_Category               0
Months_on_book              0
Total_Relationship_Count    0
Months_Inactive_12_mon      0
Contacts_Count_12_mon       0
Credit_Limit                0
Total_Revolving_Bal         0
Avg_Open_To_Buy             0
Total_Trans_Amt             0
Total_Trans_Ct              0
dtype: int64


# Creating dummy variables for categorical variables
X_train = pd.get_dummies(data=X_train, drop_first=True)
X_val = pd.get_dummies(data=X_val, drop_first=True)
X_test = pd.get_dummies(data=X_test, drop_first=True)


print("Shape of X Training set : ", X_train.shape)
print("Shape of X validation set : ", X_val.shape)
print("Shape of X test set : ", X_test.shape)
print("")

print("Shape of Y Training set : ", y_train.shape)
print("Shape of Y test set : ", y_val.shape)
print("Shape of Y test set : ", y_test.shape)
print("")

print("Percentage of classes in training set:")
print(y_train.value_counts(normalize=True))
print("")

print("Percentage of classes in validation set:")
print(y_val.value_counts(normalize=True))
print("")

print("Percentage of classes in test set:")
print(y_test.value_counts(normalize=True))

Shape of X Training set :  (6075, 44)
Shape of X validation set :  (2026, 44)
Shape of X test set :  (2026, 44)

Shape of Y Training set :  (6075,)
Shape of Y test set :  (2026,)
Shape of Y test set :  (2026,)

Percentage of classes in training set:
0    0.839342
1    0.160658
Name: Attrition_Flag, dtype: float64

Percentage of classes in validation set:
0    0.839092
1    0.160908
Name: Attrition_Flag, dtype: float64

Percentage of classes in test set:
0    0.839585
1    0.160415
Name: Attrition_Flag, dtype: float64


X_test.info()

<class 'pandas.core.frame.DataFrame'>
Int64Index: 2026 entries, 9760 to 413
Data columns (total 44 columns):
 #   Column                          Non-Null Count  Dtype  
---  ------                          --------------  -----  
 0   Customer_Age                    2026 non-null   int64  
 1   Months_on_book                  2026 non-null   int64  
 2   Credit_Limit                    2026 non-null   float64
 3   Total_Revolving_Bal             2026 non-null   int64  
 4   Avg_Open_To_Buy                 2026 non-null   float64
 5   Total_Trans_Amt                 2026 non-null   int64  
 6   Total_Trans_Ct                  2026 non-null   int64  
 7   Gender_M                        2026 non-null   uint8  
 8   Dependent_count_1               2026 non-null   uint8  
 9   Dependent_count_2               2026 non-null   uint8  
 10  Dependent_count_3               2026 non-null   uint8  
 11  Dependent_count_4               2026 non-null   uint8  
 12  Dependent_count_5               2026 non-null   uint8  
 13  Education_Level_Doctorate       2026 non-null   uint8  
 14  Education_Level_Graduate        2026 non-null   uint8  
 15  Education_Level_High School     2026 non-null   uint8  
 16  Education_Level_Post-Graduate   2026 non-null   uint8  
 17  Education_Level_Uneducated      2026 non-null   uint8  
 18  Marital_Status_Married          2026 non-null   uint8  
 19  Marital_Status_Single           2026 non-null   uint8  
 20  Income_Category_$40K - $60K     2026 non-null   uint8  
 21  Income_Category_$60K - $80K     2026 non-null   uint8  
 22  Income_Category_$80K - $120K    2026 non-null   uint8  
 23  Income_Category_Less than $40K  2026 non-null   uint8  
 24  Card_Category_Gold              2026 non-null   uint8  
 25  Card_Category_Platinum          2026 non-null   uint8  
 26  Card_Category_Silver            2026 non-null   uint8  
 27  Total_Relationship_Count_2      2026 non-null   uint8  
 28  Total_Relationship_Count_3      2026 non-null   uint8  
 29  Total_Relationship_Count_4      2026 non-null   uint8  
 30  Total_Relationship_Count_5      2026 non-null   uint8  
 31  Total_Relationship_Count_6      2026 non-null   uint8  
 32  Months_Inactive_12_mon_1        2026 non-null   uint8  
 33  Months_Inactive_12_mon_2        2026 non-null   uint8  
 34  Months_Inactive_12_mon_3        2026 non-null   uint8  
 35  Months_Inactive_12_mon_4        2026 non-null   uint8  
 36  Months_Inactive_12_mon_5        2026 non-null   uint8  
 37  Months_Inactive_12_mon_6        2026 non-null   uint8  
 38  Contacts_Count_12_mon_1         2026 non-null   uint8  
 39  Contacts_Count_12_mon_2         2026 non-null   uint8  
 40  Contacts_Count_12_mon_3         2026 non-null   uint8  
 41  Contacts_Count_12_mon_4         2026 non-null   uint8  
 42  Contacts_Count_12_mon_5         2026 non-null   uint8  
 43  Contacts_Count_12_mon_6         2026 non-null   uint8  
dtypes: float64(2), int64(5), uint8(37)
memory usage: 199.8 KB


models_list = {"Name": [], "CV_Score": [], "Model": []}
# Empty dictionery to store Recall Score Values


models1 = []  # Empty list to store all the models
# Appending models into the list
models1.append(
    ("Logistic Regression", LogisticRegression(solver="liblinear", random_state=1))
)
models1.append(("Decision Tree", DecisionTreeClassifier(random_state=1)))
models1.append(("Bagging", BaggingClassifier(random_state=1)))
models1.append(("Random Forest", RandomForestClassifier(random_state=1)))
models1.append(("Gradient Boost", GradientBoostingClassifier(random_state=1)))
models1.append(("Adaboost", AdaBoostClassifier(random_state=1)))
models1.append(("Xgboost", XGBClassifier(random_state=1, eval_metric="logloss")))

results = []  # Empty list to store all model's CV scores
names = []  # Empty list to store name of the models
score1 = []
# loop through all models to get the mean cross validated score
print("\n" "Cross-Validation Performance:" "\n")
for name, model in models1:
    scoring = "recall"
    kfold = StratifiedKFold(
        n_splits=5, shuffle=True, random_state=1
    )  # Setting number of splits equal to 5
    cv_result = cross_val_score(
        estimator=model, X=X_train, y=y_train, scoring=scoring, cv=kfold
    )
    results.append(cv_result)
    names.append(name)  
    models_list["Name"].append(name)
    models_list["CV_Score"].append(cv_result.mean() * 100)
    models_list["Model"].append(model)
    print("{}: {}".format(name, cv_result.mean() * 100))

    
print("\n" "Recall Score - Validation Performance:" "\n")

for name, model in models1:
    model.fit(X_train, y_train)
    scores = recall_score(y_val, model.predict(X_val)) * 100
    score1.append(scores)    
    print("{}: {}".format(name, scores))

Cross-Validation Performance:

Logistic Regression: 32.38147566718995
Decision Tree: 74.17687074829932
Bagging: 73.04918890633176
Random Forest: 65.36630036630036
Gradient Boost: 78.06750392464677
Adaboost: 76.9403453689168
Xgboost: 79.91313448456306

Recall Score - Validation Performance:

Logistic Regression: 50.920245398773
Decision Tree: 76.38036809815951
Bagging: 76.38036809815951
Random Forest: 69.01840490797547
Gradient Boost: 79.4478527607362
Adaboost: 80.98159509202453
Xgboost: 83.74233128834356


# Plotting boxplots for CV scores of all models defined above
fig = plt.figure(figsize=(15, 7))

fig.suptitle("Algorithm Comparison - Original Data")
ax = fig.add_subplot(111)

plt.boxplot(results)
ax.set_xticklabels(names)

plt.show()


# Fit SMOTE on train data(Synthetic Minority Oversampling Technique)
sm = SMOTE(sampling_strategy=1, k_neighbors=5, random_state=1)
X_train_over, y_train_over = sm.fit_resample(X_train, y_train)

print("Before OverSampling, count of label '1': {}".format(sum(y_train == 1)))
print("Before OverSampling, count of label '0': {} \n".format(sum(y_train == 0)))

print("After OverSampling, count of label '1': {}".format(sum(y_train_over == 1)))
print("After OverSampling, count of label '0': {} \n".format(sum(y_train_over == 0)))

print("After OverSampling, the shape of train_X: {}".format(X_train_over.shape))
print("After OverSampling, the shape of train_y: {} \n".format(y_train_over.shape))

Before OverSampling, count of label '1': 976
Before OverSampling, count of label '0': 5099 

After OverSampling, count of label '1': 5099
After OverSampling, count of label '0': 5099 

After OverSampling, the shape of train_X: (10198, 44)
After OverSampling, the shape of train_y: (10198,)


models2 = []  # Empty list to store all the models

# Appending models into the list
models2.append(
    ("Over Logistic Regression", LogisticRegression(solver="liblinear", random_state=1))
)
models2.append(("Over Decision Tree", DecisionTreeClassifier(random_state=1)))
models2.append(("Over Bagging", BaggingClassifier(random_state=1)))
models2.append(("Over Random Forest", RandomForestClassifier(random_state=1)))
models2.append(("Over Gradient Boost", GradientBoostingClassifier(random_state=1)))
models2.append(("Over Adaboost", AdaBoostClassifier(random_state=1)))
models2.append(("Over Xgboost", XGBClassifier(random_state=1, eval_metric="logloss")))

results = []  # Empty list to store all model's CV scores
names = []  # Empty list to store name of the models
score2 = []
# loop through all models to get the mean cross validated score
print("\n" "Cross-Validation Performance:" "\n")
for name, model in models2:
    scoring = "recall"
    kfold = StratifiedKFold(
        n_splits=5, shuffle=True, random_state=1
    )  # Setting number of splits equal to 5
    cv_result = cross_val_score(
        estimator=model, X=X_train_over, y=y_train_over, scoring=scoring, cv=kfold
    )
    results.append(cv_result)
    names.append(name)
    models_list["Name"].append(name)
    models_list["CV_Score"].append(cv_result.mean() * 100)
    models_list["Model"].append(model)
    print("{}: {}".format(name, cv_result.mean() * 100))

print("\n" "Recall Score - Validation Performance:" "\n")

for name, model in models2:
    model.fit(X_train_over, y_train_over)
    scores = recall_score(y_val, model.predict(X_val)) * 100
    score2.append(scores)
    print("{}: {}".format(name, scores))

Cross-Validation Performance:

Over Logistic Regression: 90.15520791240932
Over Decision Tree: 94.31268640920548
Over Bagging: 95.15626623564047
Over Random Forest: 95.74448228751756
Over Gradient Boost: 96.33281379283804
Over Adaboost: 96.1562469933999
Over Xgboost: 96.70532432796473

Recall Score - Validation Performance:

Over Logistic Regression: 58.58895705521472
Over Decision Tree: 76.68711656441718
Over Bagging: 78.52760736196319
Over Random Forest: 72.39263803680981
Over Gradient Boost: 85.2760736196319
Over Adaboost: 82.82208588957054
Over Xgboost: 85.58282208588957


# Plotting boxplots for CV scores of all models defined above
fig = plt.figure(figsize=(15, 7))

fig.suptitle("Algorithm Comparison - Oversampling Data")
ax = fig.add_subplot(111)

plt.boxplot(results)
ax.set_xticklabels(names)

plt.show()


# fit random under sampler on the train data
rus = RandomUnderSampler(random_state=1, sampling_strategy=1)
X_train_un, y_train_un = rus.fit_resample(X_train, y_train)

print("Before Under Sampling, count of label '1': {}".format(sum(y_train == 1)))
print("Before Under Sampling, count of label '0': {} \n".format(sum(y_train == 0)))

print("After Under Sampling, count of label '1': {}".format(sum(y_train_un == 1)))
print("After Under Sampling, count of label '0': {} \n".format(sum(y_train_un == 0)))

print("After Under Sampling, the shape of train_X: {}".format(X_train_un.shape))
print("After Under Sampling, the shape of train_y: {} \n".format(y_train_un.shape))

Before Under Sampling, count of label '1': 976
Before Under Sampling, count of label '0': 5099 

After Under Sampling, count of label '1': 976
After Under Sampling, count of label '0': 976 

After Under Sampling, the shape of train_X: (1952, 44)
After Under Sampling, the shape of train_y: (1952,)


models3 = []  # Empty list to store all the models

# Appending models into the list
models3.append(
    (
        "Under Logistic Regression",
        LogisticRegression(solver="liblinear", random_state=1),
    )
)
models3.append(("Under Decision Tree", DecisionTreeClassifier(random_state=1)))
models3.append(("Under Bagging", BaggingClassifier(random_state=1)))
models3.append(("Under Random Forest", RandomForestClassifier(random_state=1)))
models3.append(("Under Gradient Boost", GradientBoostingClassifier(random_state=1)))
models3.append(("Under Adaboost", AdaBoostClassifier(random_state=1)))
models3.append(("Under Xgboost", XGBClassifier(random_state=1, eval_metric="logloss")))

results = []  # Empty list to store all model's CV scores
names = []  # Empty list to store name of the models
score3 = []
# loop through all models to get the mean cross validated score
print("\n" "Cross-Validation Performance:" "\n")
for name, model in models3:
    scoring = "recall"
    kfold = StratifiedKFold(
        n_splits=5, shuffle=True, random_state=1
    )  # Setting number of splits equal to 5
    cv_result = cross_val_score(
        estimator=model, X=X_train_un, y=y_train_un, scoring=scoring, cv=kfold
    )
    results.append(cv_result)
    names.append(name)
    models_list["Name"].append(name)
    models_list["CV_Score"].append(cv_result.mean() * 100)
    models_list["Model"].append(model)
    print("{}: {}".format(name, cv_result.mean() * 100))

print("\n" "Recall Score - Validation Performance:" "\n")

for name, model in models3:
    model.fit(X_train_un, y_train_un)
    scores = recall_score(y_val, model.predict(X_val)) * 100
    score3.append(scores)
    print("{}: {}".format(name, scores))

Cross-Validation Performance:

Under Logistic Regression: 83.91732077446363
Under Decision Tree: 87.09105180533753
Under Bagging: 89.03976975405548
Under Random Forest: 90.98482469911042
Under Gradient Boost: 92.72736787022501
Under Adaboost: 91.70329670329672
Under Xgboost: 92.31711145996861

Recall Score - Validation Performance:

Under Logistic Regression: 84.66257668711657
Under Decision Tree: 89.2638036809816
Under Bagging: 89.87730061349694
Under Random Forest: 92.33128834355828
Under Gradient Boost: 91.71779141104295
Under Adaboost: 92.02453987730061
Under Xgboost: 93.86503067484662


# Plotting boxplots for CV scores of all models defined above
fig = plt.figure(figsize=(15, 7))

fig.suptitle("Algorithm Comparison - Undersampling Data")
ax = fig.add_subplot(111)

plt.boxplot(results)
ax.set_xticklabels(names)

plt.show()


models_df = pd.DataFrame(models_list)
models_df.sort_values(by=["CV_Score"], ascending=False)


%%time 

# defining model
model = AdaBoostClassifier(random_state=1)

# Parameter grid to pass in GridSearchCV

param_grid = {
    "n_estimators": np.arange(10, 110, 10),
    "learning_rate": [0.1, 0.01, 0.2, 0.05, 1],
    "base_estimator": [
        DecisionTreeClassifier(max_depth=1, random_state=1),
        DecisionTreeClassifier(max_depth=2, random_state=1),
        DecisionTreeClassifier(max_depth=3, random_state=1),
    ],
}

# Type of scoring used to compare parameter combinations
scorer = metrics.make_scorer(metrics.recall_score)

# Calling GridSearchCV
grid_cv = GridSearchCV(estimator=model, param_grid=param_grid, scoring=scorer, cv=5, n_jobs = -1)

# Fitting parameters in GridSearchCV
grid_cv.fit(X_train_over, y_train_over)

print(
    "Best Parameters:{} \nScore: {}".format(grid_cv.best_params_, grid_cv.best_score_)
)


# Set the clf to the best combination of parameters
adb_tuned1 = grid_cv.best_estimator_

# Fit the model on training data
adb_tuned1.fit(X_train_over, y_train_over)

Best Parameters:{'base_estimator': DecisionTreeClassifier(max_depth=3, random_state=1), 'learning_rate': 0.05, 'n_estimators': 30} 
Score: 0.9321518597625508
Wall time: 3min 18s

AdaBoostClassifier(base_estimator=DecisionTreeClassifier(max_depth=3,
                                                         random_state=1),
                   learning_rate=0.05, n_estimators=30, random_state=1)


# Calculating different metrics on train set
Adaboost_grid_train = model_performance_classification_sklearn_with_threshold(
    adb_tuned1, X_train_over, y_train_over
)
print("Training performance:")
print(Adaboost_grid_train)

print("*************************************")

Adaboost_grid_val = model_performance_classification_sklearn_with_threshold(
    adb_tuned1, X_val, y_val
)
print("Validation performance:")
print(Adaboost_grid_val)

Training performance:
   Accuracy    Recall  Precision        F1
0  0.939498  0.963718   0.919192  0.940929
*************************************
Validation performance:
   Accuracy    Recall  Precision        F1
0  0.911155  0.871166   0.672986  0.759358


# Creating confusion matrix
confusion_matrix_sklearn_with_threshold(adb_tuned1, X_train_over, y_train_over)

# Creating confusion matrix
confusion_matrix_sklearn_with_threshold(adb_tuned1, X_val, y_val)


%%time 

# defining model
model = AdaBoostClassifier(random_state=1)

# Parameter grid to pass in GridSearchCV

param_grid = {
    "n_estimators": np.arange(10, 110, 10),
    "learning_rate": [0.1, 0.01, 0.2, 0.05, 1],
    "base_estimator": [
        DecisionTreeClassifier(max_depth=1, random_state=1),
        DecisionTreeClassifier(max_depth=2, random_state=1),
        DecisionTreeClassifier(max_depth=3, random_state=1),
    ],
}

# Type of scoring used to compare parameter combinations
scorer = metrics.make_scorer(metrics.recall_score)

#Calling RandomizedSearchCV
randomized_cv = RandomizedSearchCV(estimator=model, param_distributions=param_grid, n_jobs = -1, n_iter=50, scoring=scorer, cv=5, random_state=1)

#Fitting parameters in RandomizedSearchCV
randomized_cv.fit(X_train_over,y_train_over)

print("Best parameters are {} with CV score={}:" .format(randomized_cv.best_params_,randomized_cv.best_score_))


# Set the clf to the best combination of parameters
adb_tuned2 = randomized_cv.best_estimator_

# Fit the model on training data
adb_tuned2.fit(X_train_over, y_train_over)

Best parameters are {'n_estimators': 10, 'learning_rate': 0.2, 'base_estimator': DecisionTreeClassifier(max_depth=3, random_state=1)} with CV score=0.9305853489580326:
Wall time: 1min 7s

AdaBoostClassifier(base_estimator=DecisionTreeClassifier(max_depth=3,
                                                         random_state=1),
                   learning_rate=0.2, n_estimators=10, random_state=1)


# Calculating different metrics on train set
Adaboost_random_train = model_performance_classification_sklearn_with_threshold(
    adb_tuned2, X_train_over, y_train_over
)
print("Training performance:")
print(Adaboost_random_train)

print("*************************************")

Adaboost_random_val = model_performance_classification_sklearn_with_threshold(
    adb_tuned2, X_val, y_val
)
print("Validation performance:")
print(Adaboost_random_val)

Training performance:
   Accuracy    Recall  Precision        F1
0   0.94695  0.964699   0.931629  0.947876
*************************************
Validation performance:
   Accuracy    Recall  Precision        F1
0  0.916091  0.855828    0.69403  0.766484


# Creating confusion matrix
confusion_matrix_sklearn_with_threshold(adb_tuned2, X_train_over, y_train_over)

# Creating confusion matrix
confusion_matrix_sklearn_with_threshold(adb_tuned2, X_val, y_val)


%%time 

# defining model
model = GradientBoostingClassifier(random_state=1)

# Parameter grid to pass in GridSearchCV

param_grid = {
    "n_estimators": [100,150,200,250],
    "subsample":[0.8,0.9,1],
    "max_features":[0.7,0.8,0.9,1],
    "max_depth": [3, 5, 7],
}

# Type of scoring used to compare parameter combinations
scorer = metrics.make_scorer(metrics.recall_score)

# Calling GridSearchCV
grid_cv = GridSearchCV(estimator=model, param_grid=param_grid, scoring=scorer, cv=5, n_jobs = -1)

# Fitting parameters in GridSearchCV
grid_cv.fit(X_train_over, y_train_over)

print(
    "Best Parameters:{} \nScore: {}".format(grid_cv.best_params_, grid_cv.best_score_)
)


# Set the clf to the best combination of parameters
gb_tuned1 = grid_cv.best_estimator_

# Fit the model on training data
gb_tuned1.fit(X_train_over, y_train_over)

Best Parameters:{'max_depth': 7, 'max_features': 0.9, 'n_estimators': 100, 'subsample': 1} 
Score: 0.9149015759395024
Wall time: 13min 58s

GradientBoostingClassifier(max_depth=7, max_features=0.9, random_state=1,
                           subsample=1)


# Calculating different metrics on train set
GB_grid_train = model_performance_classification_sklearn_with_threshold(
    gb_tuned1, X_train_over, y_train_over
)
print("Training performance:")
print(GB_grid_train)

print("*************************************")

GB_grid_val = model_performance_classification_sklearn_with_threshold(
    gb_tuned1, X_val, y_val
)
print("Validation performance:")
print(GB_grid_val)

Training performance:
   Accuracy  Recall  Precision   F1
0       1.0     1.0        1.0  1.0
*************************************
Validation performance:
   Accuracy    Recall  Precision        F1
0  0.950642  0.849693   0.844512  0.847095


# Creating confusion matrix
confusion_matrix_sklearn_with_threshold(gb_tuned1, X_train_over, y_train_over)

# Creating confusion matrix
confusion_matrix_sklearn_with_threshold(gb_tuned1, X_val, y_val)


%%time 

# defining model
model = GradientBoostingClassifier(random_state=1)

# Parameter grid to pass in GridSearchCV

param_grid = {
    "n_estimators": [100,150,200,250],
    "subsample":[0.8,0.9,1],
    "max_features":[0.7,0.8,0.9,1],
    "max_depth": [3, 5, 7],
}

# Type of scoring used to compare parameter combinations
scorer = metrics.make_scorer(metrics.recall_score)

#Calling RandomizedSearchCV
randomized_cv = RandomizedSearchCV(estimator=model, param_distributions=param_grid, n_jobs = -1, n_iter=50, scoring=scorer, cv=5, random_state=1)

#Fitting parameters in RandomizedSearchCV
randomized_cv.fit(X_train_over,y_train_over)

print("Best parameters are {} with CV score={}:" .format(randomized_cv.best_params_,randomized_cv.best_score_))

# Set the clf to the best combination of parameters
gb_tuned2 = randomized_cv.best_estimator_

# Fit the model on training data
gb_tuned2.fit(X_train_over, y_train_over)

Best parameters are {'subsample': 1, 'n_estimators': 100, 'max_features': 0.7, 'max_depth': 7} with CV score=0.9129405992033712:
Wall time: 4min 18s

GradientBoostingClassifier(max_depth=7, max_features=0.7, random_state=1,
                           subsample=1)


# Calculating different metrics on train set
GB_random_train = model_performance_classification_sklearn_with_threshold(
    gb_tuned2, X_train_over, y_train_over
)
print("Training performance:")
print(GB_random_train)

print("*************************************")

GB_random_val = model_performance_classification_sklearn_with_threshold(
    gb_tuned2, X_val, y_val
)
print("Validation performance:")
print(GB_random_val)

Training performance:
   Accuracy    Recall  Precision        F1
0  0.999902  0.999804        1.0  0.999902
*************************************
Validation performance:
   Accuracy    Recall  Precision        F1
0  0.950148  0.837423   0.850467  0.843895


# Creating confusion matrix
confusion_matrix_sklearn_with_threshold(gb_tuned2, X_train_over, y_train_over)

# Creating confusion matrix
confusion_matrix_sklearn_with_threshold(gb_tuned2, X_val, y_val)


%%time 

#defining model
model = XGBClassifier(random_state=1,eval_metric='logloss')

#Parameter grid to pass in GridSearchCV
param_grid={
        'n_estimators':np.arange(50,150,50),
            'scale_pos_weight':[5,10],
            'learning_rate':[0.01,0.1],
            'gamma':[0,1,3],
            'subsample':[0.8,0.9,1],
            'max_depth':np.arange(1,5,1),
}


# Type of scoring used to compare parameter combinations
scorer = metrics.make_scorer(metrics.recall_score)

#Calling GridSearchCV
grid_cv = GridSearchCV(estimator=model, param_grid=param_grid, scoring=scorer, cv=5, n_jobs = -1, verbose= 2)

#Fitting parameters in GridSearchCV
grid_cv.fit(X_train_over,y_train_over)


print("Best parameters are {} with CV score={}:" .format(grid_cv.best_params_,grid_cv.best_score_))

xgb_tuned1 = grid_cv.best_estimator_

xgb_tuned1.fit(X_train_over, y_train_over)

Fitting 5 folds for each of 288 candidates, totalling 1440 fits
Best parameters are {'gamma': 0, 'learning_rate': 0.01, 'max_depth': 2, 'n_estimators': 100, 'scale_pos_weight': 10, 'subsample': 0.8} with CV score=0.9986274509803922:
Wall time: 10min 5s

XGBClassifier(base_score=0.5, booster='gbtree', callbacks=None,
              colsample_bylevel=1, colsample_bynode=1, colsample_bytree=1,
              early_stopping_rounds=None, enable_categorical=False,
              eval_metric='logloss', gamma=0, gpu_id=-1,
              grow_policy='depthwise', importance_type=None,
              interaction_constraints='', learning_rate=0.01, max_bin=256,
              max_cat_to_onehot=4, max_delta_step=0, max_depth=2, max_leaves=0,
              min_child_weight=1, missing=nan, monotone_constraints='()',
              n_estimators=100, n_jobs=0, num_parallel_tree=1, predictor='auto',
              random_state=1, reg_alpha=0, reg_lambda=1, ...)


# Calculating different metrics on train set
xgboost_grid_train = model_performance_classification_sklearn_with_threshold(
    xgb_tuned1, X_train_over, y_train_over
)
print("Training performance:")
print(xgboost_grid_train)

print("*************************************")

xgboost_grid_val = model_performance_classification_sklearn_with_threshold(
    xgb_tuned1, X_val, y_val
)
print("Validation performance:")
print(xgboost_grid_train)

Training performance:
   Accuracy    Recall  Precision        F1
0  0.707197  0.999804   0.630706  0.773479
*************************************
Validation performance:
   Accuracy    Recall  Precision        F1
0  0.707197  0.999804   0.630706  0.773479


# Creating confusion matrix
confusion_matrix_sklearn_with_threshold(xgb_tuned1, X_train_over, y_train_over)

# Creating confusion matrix
confusion_matrix_sklearn_with_threshold(xgb_tuned1, X_val, y_val)


%%time

# defining model
model = XGBClassifier(random_state=1,eval_metric='logloss')

# Parameter grid to pass in RandomizedSearchCV
param_grid={
        'n_estimators':np.arange(50,150,50),
            'scale_pos_weight':[5,10],
            'learning_rate':[0.01,0.1],
            'gamma':[0,1,3],
            'subsample':[0.8,0.9,1],
            'max_depth':np.arange(1,5,1),
}

# Type of scoring used to compare parameter combinations
scorer = metrics.make_scorer(metrics.recall_score)

#Calling RandomizedSearchCV
randomized_cv = RandomizedSearchCV(estimator=model, param_distributions=param_grid, n_iter=50, scoring=scorer, cv=5, random_state=1, n_jobs = -1)

#Fitting parameters in RandomizedSearchCV
randomized_cv.fit(X_train_over,y_train_over)

print("Best parameters are {} with CV score={}:" .format(randomized_cv.best_params_,randomized_cv.best_score_))

xgb_tuned2 = randomized_cv.best_estimator_

xgb_tuned2.fit(X_train_over, y_train_over)

Best parameters are {'subsample': 1, 'scale_pos_weight': 10, 'n_estimators': 50, 'max_depth': 1, 'learning_rate': 0.1, 'gamma': 3} with CV score=0.9972549019607844:
Wall time: 1min 37s

XGBClassifier(base_score=0.5, booster='gbtree', callbacks=None,
              colsample_bylevel=1, colsample_bynode=1, colsample_bytree=1,
              early_stopping_rounds=None, enable_categorical=False,
              eval_metric='logloss', gamma=3, gpu_id=-1,
              grow_policy='depthwise', importance_type=None,
              interaction_constraints='', learning_rate=0.1, max_bin=256,
              max_cat_to_onehot=4, max_delta_step=0, max_depth=1, max_leaves=0,
              min_child_weight=1, missing=nan, monotone_constraints='()',
              n_estimators=50, n_jobs=0, num_parallel_tree=1, predictor='auto',
              random_state=1, reg_alpha=0, reg_lambda=1, ...)


# Calculating different metrics on train set
xgboost_random_train = model_performance_classification_sklearn_with_threshold(
    xgb_tuned2, X_train_over, y_train_over
)
print("Training performance:")
print(xgboost_random_train)

print("*************************************")

xgboost_random_val = model_performance_classification_sklearn_with_threshold(
    xgb_tuned2, X_val, y_val
)
print("Validation performance:")
print(xgboost_random_val)

Training performance:
   Accuracy    Recall  Precision        F1
0  0.651696  0.998431   0.589577  0.741372
*************************************
Validation performance:
   Accuracy   Recall  Precision        F1
0  0.405726  0.98773   0.211564  0.348485


# Creating confusion matrix
confusion_matrix_sklearn_with_threshold(xgb_tuned2, X_train_over, y_train_over)

# Creating confusion matrix
confusion_matrix_sklearn_with_threshold(xgb_tuned2, X_val, y_val)


# training performance comparison

models_train_comp_df = pd.concat(
    [
        Adaboost_grid_train.T,
        Adaboost_random_train.T,
        GB_grid_train.T,
        GB_random_train.T,
        xgboost_grid_train.T,
        xgboost_random_train.T,
    ],
    axis=1,
)
models_train_comp_df.columns = [
    "Adaboost Grid",
    "Adaboost Random",
    "Gradient Grid",
    "Gradient Random",
    "XGBoost Grid",
    "XGBoost Random",
]
print("Training performance comparison:")
models_train_comp_df

Training performance comparison:


# training performance comparison

models_train_comp_df = pd.concat(
    [
        Adaboost_grid_val.T,
        Adaboost_random_val.T,
        GB_grid_val.T,
        GB_random_val.T,
        xgboost_grid_val.T,
        xgboost_random_val.T,
    ],
    axis=1,
)
models_train_comp_df.columns = [
    "Adaboost Grid",
    "Adaboost Random",
    "Gradient Grid",
    "Gradient Random",
    "XGBoost Grid",
    "XGBoost Random",
]
print("Validation performance comparison:")
models_train_comp_df

Validation performance comparison:


# Calculating different metrics on the test set
AdaBoost_Grid_test = model_performance_classification_sklearn_with_threshold(
    adb_tuned1, X_test, y_test
)

# Calculating different metrics on the test set
AdaBoost_Random_test = model_performance_classification_sklearn_with_threshold(
    adb_tuned2, X_test, y_test
)

# Calculating different metrics on the test set
GB_Grid_test = model_performance_classification_sklearn_with_threshold(
    gb_tuned1, X_test, y_test
)

# Calculating different metrics on the test set
GB_Random_test = model_performance_classification_sklearn_with_threshold(
    gb_tuned2, X_test, y_test
)

# Calculating different metrics on the test set
XGB_Grid_test = model_performance_classification_sklearn_with_threshold(
    xgb_tuned1, X_test, y_test
)

# Calculating different metrics on the test set
XGB_Random_test = model_performance_classification_sklearn_with_threshold(
    xgb_tuned2, X_test, y_test
)


# training performance comparison

models_train_comp_df = pd.concat(
    [
        Adaboost_grid_train.T,
        Adaboost_random_train.T,
        GB_grid_train.T,
        GB_random_train.T,
        xgboost_grid_train.T,
        xgboost_random_train.T,
    ],
    axis=1,
)
models_train_comp_df.columns = [
    "Adaboost Grid",
    "Adaboost Random",
    "Gradient Grid",
    "Gradient Random",
    "XGBoost Grid",
    "XGBoost Random",
]
print("Training performance comparison:")
print(models_train_comp_df)

# Testing performance comparison

models_train_comp_df = pd.concat(
    [
        AdaBoost_Grid_test.T,
        AdaBoost_Random_test.T,
        GB_Grid_test.T,
        GB_Random_test.T,
        XGB_Grid_test.T,
        XGB_Random_test.T,
    ],
    axis=1,
)
models_train_comp_df.columns = [
    "Adaboost Grid",
    "Adaboost Random",
    "Gradient Grid",
    "Gradient Random",
    "XGBoost Grid",
    "XGBoost Random",
]

print("\n\n")
print("Training performance comparison:")
print(models_train_comp_df)

Training performance comparison:
           Adaboost Grid  Adaboost Random  Gradient Grid  Gradient Random  \
Accuracy        0.939498         0.946950            1.0         0.999902   
Recall          0.963718         0.964699            1.0         0.999804   
Precision       0.919192         0.931629            1.0         1.000000   
F1              0.940929         0.947876            1.0         0.999902   

           XGBoost Grid  XGBoost Random  
Accuracy       0.707197        0.651696  
Recall         0.999804        0.998431  
Precision      0.630706        0.589577  
F1             0.773479        0.741372  


Training performance comparison:
           Adaboost Grid  Adaboost Random  Gradient Grid  Gradient Random  \
Accuracy        0.905726         0.917078       0.958539         0.960020   
Recall          0.898462         0.889231       0.886154         0.883077   
Precision       0.648889         0.686461       0.859701         0.869697   
F1              0.753548         0.774799       0.872727         0.876336   

           XGBoost Grid  XGBoost Random  
Accuracy       0.496545        0.394867  
Recall         1.000000        0.996923  
Precision      0.241636        0.209167  
F1             0.389222        0.345784


### Feature Importance Using Sklearn


feature_names = X_test.columns
importances = adb_tuned2.feature_importances_
indices = np.argsort(importances)

plt.figure(figsize=(12, 12))
plt.title("Feature Importances")
plt.barh(range(len(indices)), importances[indices], color="violet", align="center")
plt.yticks(range(len(indices)), [feature_names[i] for i in indices])
plt.xlabel("Relative Importance")
plt.show()


# Separating target variable and other variables
pX = df.drop(["Attrition_Flag"], axis=1)
pY = df["Attrition_Flag"]


pX.drop(["AgeGroup"], axis=1, inplace=True)
pX.drop(["Months_on_book_Grp"], axis=1, inplace=True)
pX.drop(["Credit_Limit_Grp"], axis=1, inplace=True)


# Identofying the category columns
category_columnNames = pX.describe(include=["category"]).columns
category_columnNames

Index(['Gender', 'Dependent_count', 'Education_Level', 'Marital_Status',
       'Income_Category', 'Card_Category', 'Total_Relationship_Count',
       'Months_Inactive_12_mon', 'Contacts_Count_12_mon'],
      dtype='object')


# Identifying the numerical columns
number_columnNames = (
    pX.describe(include=["int64"]).columns.tolist()
    + pX.describe(include=["float64"]).columns.tolist()
)
number_columnNames

['Customer_Age',
 'Months_on_book',
 'Total_Revolving_Bal',
 'Total_Trans_Amt',
 'Total_Trans_Ct',
 'Credit_Limit',
 'Avg_Open_To_Buy',
 'Total_Amt_Chng_Q4_Q1',
 'Total_Ct_Chng_Q4_Q1',
 'Avg_Utilization_Ratio']


# creating a transformer for numerical variables, which will apply simple imputer on the numerical variables
numeric_transformer = Pipeline(steps=[("imputer", SimpleImputer(strategy="median"))])

# creating a transformer for categorical variables, which will first apply simple imputer and 
#then do one hot encoding for categorical variables
categorical_transformer = Pipeline(
    steps=[
        ("imputer", SimpleImputer(strategy="most_frequent")),
        ("onehot", OneHotEncoder(handle_unknown="ignore")),
    ]
)
# handle_unknown = "ignore", allows model to handle any unknown category in the test data

# combining categorical transformer and numerical transformer using a column transformer
preprocessor = ColumnTransformer(
    transformers=[
        ("num", numeric_transformer, number_columnNames),
        ("cat", categorical_transformer, category_columnNames),
    ],
    remainder="passthrough",
)
# remainder = "passthrough" has been used, it will allow variables that are present in original data 
# but not in "numerical_columns" and "categorical_columns" to pass through the column transformer without any changes


# Splitting the data into train and test sets
XX_train, XX_test, yy_train, yy_test = train_test_split(
    pX, pY, test_size=0.30, random_state=1, stratify=y
)
print(XX_train.shape, XX_test.shape)

(7088, 19) (3039, 19)


# Creating new pipeline with best parameters
model = Pipeline(
    steps=[
        ("pre", preprocessor),
        (
            "AB",
            AdaBoostClassifier(
                random_state=1,
                n_estimators=10,
                learning_rate=0.2,
                base_estimator=DecisionTreeClassifier(max_depth=3, random_state=1),
            ),
        ),
    ]
)
# Fit the model on training data
model.fit(XX_train, yy_train)

Pipeline(steps=[('pre',
                 ColumnTransformer(remainder='passthrough',
                                   transformers=[('num',
                                                  Pipeline(steps=[('imputer',
                                                                   SimpleImputer(strategy='median'))]),
                                                  ['Customer_Age',
                                                   'Months_on_book',
                                                   'Total_Revolving_Bal',
                                                   'Total_Trans_Amt',
                                                   'Total_Trans_Ct',
                                                   'Credit_Limit',
                                                   'Avg_Open_To_Buy',
                                                   'Total_Amt_Chng_Q4_Q1',
                                                   'Total_Ct_Chng_Q4_Q1',
                                                   'Avg_Utilization_Ratio']),
                                                 ('cat',...
                                                                   OneHotEncoder(handle_unknown='ignore'))]),
                                                  Index(['Gender', 'Dependent_count', 'Education_Level', 'Marital_Status',
       'Income_Category', 'Card_Category', 'Total_Relationship_Count',
       'Months_Inactive_12_mon', 'Contacts_Count_12_mon'],
      dtype='object'))])),
                ('AB',
                 AdaBoostClassifier(base_estimator=DecisionTreeClassifier(max_depth=3,
                                                                          random_state=1),
                                    learning_rate=0.2, n_estimators=10,
                                    random_state=1))])


XX_test["model_predictions"] = model.predict(XX_test)


XX_test[XX_test["model_predictions"] == 1]

	count	mean	std	min	25%	50%	75%	max
Customer_Age	10127.0	46.325960	8.016814	26.0	41.000	46.000	52.000	73.000
Months_on_book	10127.0	35.928409	7.986416	13.0	31.000	36.000	40.000	56.000
Credit_Limit	10127.0	8631.953698	9088.776650	1438.3	2555.000	4549.000	11067.500	34516.000
Total_Revolving_Bal	10127.0	1162.814061	814.987335	0.0	359.000	1276.000	1784.000	2517.000
Avg_Open_To_Buy	10127.0	7469.139637	9090.685324	3.0	1324.500	3474.000	9859.000	34516.000
Total_Amt_Chng_Q4_Q1	10127.0	0.759941	0.219207	0.0	0.631	0.736	0.859	3.397
Total_Trans_Amt	10127.0	4404.086304	3397.129254	510.0	2155.500	3899.000	4741.000	18484.000
Total_Trans_Ct	10127.0	64.858695	23.472570	10.0	45.000	67.000	81.000	139.000
Total_Ct_Chng_Q4_Q1	10127.0	0.712222	0.238086	0.0	0.582	0.702	0.818	3.714
Avg_Utilization_Ratio	10127.0	0.274894	0.275691	0.0	0.023	0.176	0.503	0.999

	count	mean	std	min	25%	50%	75%	max
Customer_Age	10127.0	46.325960	8.016814	26.0	41.000	46.000	52.000	73.000
Months_on_book	10127.0	35.928409	7.986416	13.0	31.000	36.000	40.000	56.000
Credit_Limit	10127.0	8631.953698	9088.776650	1438.3	2555.000	4549.000	11067.500	34516.000
Total_Revolving_Bal	10127.0	1162.814061	814.987335	0.0	359.000	1276.000	1784.000	2517.000
Avg_Open_To_Buy	10127.0	7469.139637	9090.685324	3.0	1324.500	3474.000	9859.000	34516.000
Total_Amt_Chng_Q4_Q1	10127.0	0.759941	0.219207	0.0	0.631	0.736	0.859	3.397
Total_Trans_Amt	10127.0	4404.086304	3397.129254	510.0	2155.500	3899.000	4741.000	18484.000
Total_Trans_Ct	10127.0	64.858695	23.472570	10.0	45.000	67.000	81.000	139.000
Total_Ct_Chng_Q4_Q1	10127.0	0.712222	0.238086	0.0	0.582	0.702	0.818	3.714
Avg_Utilization_Ratio	10127.0	0.274894	0.275691	0.0	0.023	0.176	0.503	0.999

	count	mean	std	min	25%	50%	75%	max
Customer_Age	10127.0	46.325960	8.016814	26.000	41.000	46.000	52.000	73.000
Months_on_book	10127.0	35.928409	7.986416	13.000	31.000	36.000	40.000	56.000
Credit_Limit	10127.0	6528.924665	5023.257606	1438.300	2555.000	4507.000	9435.000	19999.000
Total_Revolving_Bal	10127.0	1162.814061	814.987335	0.000	359.000	1276.000	1784.000	2517.000
Avg_Open_To_Buy	10127.0	5453.166224	5172.994080	3.000	1324.500	3440.000	8436.000	19995.000
Total_Amt_Chng_Q4_Q1	10127.0	0.751387	0.184542	0.289	0.631	0.736	0.859	1.201
Total_Trans_Amt	10127.0	4404.086304	3397.129254	510.000	2155.500	3899.000	4741.000	18484.000
Total_Trans_Ct	10127.0	64.858695	23.472570	10.000	45.000	67.000	81.000	139.000
Total_Ct_Chng_Q4_Q1	10127.0	0.703484	0.197203	0.228	0.582	0.702	0.818	1.172
Avg_Utilization_Ratio	10127.0	0.274894	0.275691	0.000	0.023	0.176	0.503	0.999

	CLIENTNUM	Attrition_Flag	Customer_Age	Gender	Dependent_count	Education_Level	Marital_Status	Income_Category	Card_Category	Months_on_book	Total_Relationship_Count	Months_Inactive_12_mon	Contacts_Count_12_mon	Credit_Limit	Total_Revolving_Bal	Avg_Open_To_Buy	Total_Amt_Chng_Q4_Q1	Total_Trans_Amt	Total_Trans_Ct	Total_Ct_Chng_Q4_Q1	Avg_Utilization_Ratio
0	768805383	Existing Customer	45	M	3	High School	Married	$60K - $80K	Blue	39	5	1	3	12691.0	777	11914.0	1.335	1144	42	1.625	0.061
1	818770008	Existing Customer	49	F	5	Graduate	Single	Less than $40K	Blue	44	6	1	2	8256.0	864	7392.0	1.541	1291	33	3.714	0.105
2	713982108	Existing Customer	51	M	3	Graduate	Married	$80K - $120K	Blue	36	4	1	0	3418.0	0	3418.0	2.594	1887	20	2.333	0.000
3	769911858	Existing Customer	40	F	4	High School	NaN	Less than $40K	Blue	34	3	4	1	3313.0	2517	796.0	1.405	1171	20	2.333	0.760
4	709106358	Existing Customer	40	M	3	Uneducated	Married	$60K - $80K	Blue	21	5	1	0	4716.0	0	4716.0	2.175	816	28	2.500	0.000

	CLIENTNUM	Attrition_Flag	Customer_Age	Gender	Dependent_count	Education_Level	Marital_Status	Income_Category	Card_Category	Months_on_book	Total_Relationship_Count	Months_Inactive_12_mon	Contacts_Count_12_mon	Credit_Limit	Total_Revolving_Bal	Avg_Open_To_Buy	Total_Amt_Chng_Q4_Q1	Total_Trans_Amt	Total_Trans_Ct	Total_Ct_Chng_Q4_Q1	Avg_Utilization_Ratio
10122	772366833	Existing Customer	50	M	2	Graduate	Single	$40K - $60K	Blue	40	3	2	3	4003.0	1851	2152.0	0.703	15476	117	0.857	0.462
10123	710638233	Attrited Customer	41	M	2	NaN	Divorced	$40K - $60K	Blue	25	4	2	3	4277.0	2186	2091.0	0.804	8764	69	0.683	0.511
10124	716506083	Attrited Customer	44	F	1	High School	Married	Less than $40K	Blue	36	5	3	4	5409.0	0	5409.0	0.819	10291	60	0.818	0.000
10125	717406983	Attrited Customer	30	M	2	Graduate	NaN	$40K - $60K	Blue	36	4	3	3	5281.0	0	5281.0	0.535	8395	62	0.722	0.000
10126	714337233	Attrited Customer	43	F	2	Graduate	Married	Less than $40K	Silver	25	6	2	4	10388.0	1961	8427.0	0.703	10294	61	0.649	0.189

	count	unique	top	freq	mean	std	min	25%	50%	75%	max
CLIENTNUM	10127.0	NaN	NaN	NaN	739177606.333663	36903783.450231	708082083.0	713036770.5	717926358.0	773143533.0	828343083.0
Attrition_Flag	10127	2	Existing Customer	8500	NaN	NaN	NaN	NaN	NaN	NaN	NaN
Customer_Age	10127.0	NaN	NaN	NaN	46.32596	8.016814	26.0	41.0	46.0	52.0	73.0
Gender	10127	2	F	5358	NaN	NaN	NaN	NaN	NaN	NaN	NaN
Dependent_count	10127.0	NaN	NaN	NaN	2.346203	1.298908	0.0	1.0	2.0	3.0	5.0
Education_Level	8608	6	Graduate	3128	NaN	NaN	NaN	NaN	NaN	NaN	NaN
Marital_Status	9378	3	Married	4687	NaN	NaN	NaN	NaN	NaN	NaN	NaN
Income_Category	10127	6	Less than $40K	3561	NaN	NaN	NaN	NaN	NaN	NaN	NaN
Card_Category	10127	4	Blue	9436	NaN	NaN	NaN	NaN	NaN	NaN	NaN
Months_on_book	10127.0	NaN	NaN	NaN	35.928409	7.986416	13.0	31.0	36.0	40.0	56.0
Total_Relationship_Count	10127.0	NaN	NaN	NaN	3.81258	1.554408	1.0	3.0	4.0	5.0	6.0
Months_Inactive_12_mon	10127.0	NaN	NaN	NaN	2.341167	1.010622	0.0	2.0	2.0	3.0	6.0
Contacts_Count_12_mon	10127.0	NaN	NaN	NaN	2.455317	1.106225	0.0	2.0	2.0	3.0	6.0
Credit_Limit	10127.0	NaN	NaN	NaN	8631.953698	9088.77665	1438.3	2555.0	4549.0	11067.5	34516.0
Total_Revolving_Bal	10127.0	NaN	NaN	NaN	1162.814061	814.987335	0.0	359.0	1276.0	1784.0	2517.0
Avg_Open_To_Buy	10127.0	NaN	NaN	NaN	7469.139637	9090.685324	3.0	1324.5	3474.0	9859.0	34516.0
Total_Amt_Chng_Q4_Q1	10127.0	NaN	NaN	NaN	0.759941	0.219207	0.0	0.631	0.736	0.859	3.397
Total_Trans_Amt	10127.0	NaN	NaN	NaN	4404.086304	3397.129254	510.0	2155.5	3899.0	4741.0	18484.0
Total_Trans_Ct	10127.0	NaN	NaN	NaN	64.858695	23.47257	10.0	45.0	67.0	81.0	139.0
Total_Ct_Chng_Q4_Q1	10127.0	NaN	NaN	NaN	0.712222	0.238086	0.0	0.582	0.702	0.818	3.714
Avg_Utilization_Ratio	10127.0	NaN	NaN	NaN	0.274894	0.275691	0.0	0.023	0.176	0.503	0.999

	count	unique	top	freq	mean	std	min	25%	50%	75%	max
Attrition_Flag	10127.0	2.0	0.0	8500.0	NaN	NaN	NaN	NaN	NaN	NaN	NaN
Customer_Age	10127.0	NaN	NaN	NaN	46.32596	8.016814	26.0	41.0	46.0	52.0	73.0
Gender	10127	2	F	5358	NaN	NaN	NaN	NaN	NaN	NaN	NaN
Dependent_count	10127.0	6.0	3.0	2732.0	NaN	NaN	NaN	NaN	NaN	NaN	NaN
Education_Level	8608	6	Graduate	3128	NaN	NaN	NaN	NaN	NaN	NaN	NaN
Marital_Status	9378	3	Married	4687	NaN	NaN	NaN	NaN	NaN	NaN	NaN
Income_Category	9015	5	Less than $40K	3561	NaN	NaN	NaN	NaN	NaN	NaN	NaN
Card_Category	10127	4	Blue	9436	NaN	NaN	NaN	NaN	NaN	NaN	NaN
Months_on_book	10127.0	NaN	NaN	NaN	35.928409	7.986416	13.0	31.0	36.0	40.0	56.0
Total_Relationship_Count	10127.0	6.0	3.0	2305.0	NaN	NaN	NaN	NaN	NaN	NaN	NaN
Months_Inactive_12_mon	10127.0	7.0	3.0	3846.0	NaN	NaN	NaN	NaN	NaN	NaN	NaN
Contacts_Count_12_mon	10127.0	7.0	3.0	3380.0	NaN	NaN	NaN	NaN	NaN	NaN	NaN
Credit_Limit	10127.0	NaN	NaN	NaN	8631.953698	9088.77665	1438.3	2555.0	4549.0	11067.5	34516.0
Total_Revolving_Bal	10127.0	NaN	NaN	NaN	1162.814061	814.987335	0.0	359.0	1276.0	1784.0	2517.0
Avg_Open_To_Buy	10127.0	NaN	NaN	NaN	7469.139637	9090.685324	3.0	1324.5	3474.0	9859.0	34516.0
Total_Amt_Chng_Q4_Q1	10127.0	NaN	NaN	NaN	0.759941	0.219207	0.0	0.631	0.736	0.859	3.397
Total_Trans_Amt	10127.0	NaN	NaN	NaN	4404.086304	3397.129254	510.0	2155.5	3899.0	4741.0	18484.0
Total_Trans_Ct	10127.0	NaN	NaN	NaN	64.858695	23.47257	10.0	45.0	67.0	81.0	139.0
Total_Ct_Chng_Q4_Q1	10127.0	NaN	NaN	NaN	0.712222	0.238086	0.0	0.582	0.702	0.818	3.714
Avg_Utilization_Ratio	10127.0	NaN	NaN	NaN	0.274894	0.275691	0.0	0.023	0.176	0.503	0.999
AgeGroup	10127	6	Less_than_50	4561	NaN	NaN	NaN	NaN	NaN	NaN	NaN
Months_on_book_Grp	10127	4	Between_3-4_Year	5508	NaN	NaN	NaN	NaN	NaN	NaN	NaN
Credit_Limit_Grp	10127	6	<5K	5358	NaN	NaN	NaN	NaN	NaN	NaN	NaN

	Credit_Limit
Income_Category
$120K +	18442.0
$40K - $60K	3682.0
$60K - $80K	7660.0
$80K - $120K	12830.0
Less than $40K	2766.0

	Credit_Limit
Card_Category
Blue	4105.0
Gold	34516.0
Platinum	34516.0
Silver	29808.0

	Avg_Open_To_Buy
Income_Category
$120K +	17117.0
$40K - $60K	2580.5
$60K - $80K	6418.5
$80K - $120K	11606.0
Less than $40K	1478.0

	Name	CV_Score	Model
13	Over Xgboost	96.705324	XGBClassifier(base_score=0.5, booster='gbtree'...
11	Over Gradient Boost	96.332814	([DecisionTreeRegressor(criterion='friedman_ms...
12	Over Adaboost	96.156247	(DecisionTreeClassifier(max_depth=1, random_st...
10	Over Random Forest	95.744482	(DecisionTreeClassifier(max_features='auto', r...
9	Over Bagging	95.156266	(DecisionTreeClassifier(random_state=102886208...
8	Over Decision Tree	94.312686	DecisionTreeClassifier(random_state=1)
18	Under Gradient Boost	92.727368	([DecisionTreeRegressor(criterion='friedman_ms...
20	Under Xgboost	92.317111	XGBClassifier(base_score=0.5, booster='gbtree'...
19	Under Adaboost	91.703297	(DecisionTreeClassifier(max_depth=1, random_st...
17	Under Random Forest	90.984825	(DecisionTreeClassifier(max_features='auto', r...
7	Over Logistic Regression	90.155208	LogisticRegression(random_state=1, solver='lib...
16	Under Bagging	89.039770	(DecisionTreeClassifier(random_state=102886208...
15	Under Decision Tree	87.091052	DecisionTreeClassifier(random_state=1)
14	Under Logistic Regression	83.917321	LogisticRegression(random_state=1, solver='lib...
6	Xgboost	79.913134	XGBClassifier(base_score=0.5, booster='gbtree'...
4	Gradient Boost	78.067504	([DecisionTreeRegressor(criterion='friedman_ms...
5	Adaboost	76.940345	(DecisionTreeClassifier(max_depth=1, random_st...
1	Decision Tree	74.176871	DecisionTreeClassifier(random_state=1)
2	Bagging	73.049189	(DecisionTreeClassifier(random_state=102886208...
3	Random Forest	65.366300	(DecisionTreeClassifier(max_features='auto', r...
0	Logistic Regression	32.381476	LogisticRegression(random_state=1, solver='lib...

	Adaboost Grid	Adaboost Random	Gradient Grid	Gradient Random	XGBoost Grid	XGBoost Random
Accuracy	0.939498	0.946950	1.0	0.999902	0.707197	0.651696
Recall	0.963718	0.964699	1.0	0.999804	0.999804	0.998431
Precision	0.919192	0.931629	1.0	1.000000	0.630706	0.589577
F1	0.940929	0.947876	1.0	0.999902	0.773479	0.741372

	Adaboost Grid	Adaboost Random	Gradient Grid	Gradient Random	XGBoost Grid	XGBoost Random
Accuracy	0.911155	0.916091	0.950642	0.950148	0.530109	0.405726
Recall	0.871166	0.855828	0.849693	0.837423	1.000000	0.987730
Precision	0.672986	0.694030	0.844512	0.850467	0.255086	0.211564
F1	0.759358	0.766484	0.847095	0.843895	0.406484	0.348485

Project - Credit Card Users Churn Prediction¶

Context:¶

Objective:¶

Data Information¶

Table of Contents (TOC)¶

- Importing Packages¶

- Unwrapping Customer Information¶

- Data Pre-Processing & Sanity Checks¶

- Summary of Data Analysis¶

- EDA Analysis¶

- Model Building¶

- Model Analysis - Original Data¶

- Model Analysis - Oversampling data¶

- Model Analysis - Undersampling data¶

- Comparison Models with Data - Original vs Oversample vs Undersample ¶

- Model Performance on Test dataset¶

- Pipelines for productionizing the model ¶

- Recommendations¶

Importing required Packages:¶

Unwrapping the Customer Information:¶

Data Preprocessing & Sanity Checks¶

Dropping the Customer ID Column¶

Checking for Duplicates¶

Checking for Columns with missing values¶

Validating the values of the columns to observe the pattern and data correctness¶

Replacing the incorrect value for Income Category type¶

Replacing the Attrition Flag text to binary¶

Summary of Data Analysis¶

Common Functions¶

EDA Analysis - Analyzing respective attributes to understand the data pattern¶

Analyzing the count and percentage of Categorical attributes using a bar chart¶

Insights from Categorical Data¶

Analyzing the Numerical attributes using Histogram and Box Plots¶

Insights from Numerical Data¶

Univariate Analysis¶

Aalyzing the Age of the Customers¶

Analyzing the Month (period) of relationship with the bank¶

Analyzing the Credit Limit¶

Analyzing the Total_Revolving_Bal¶

Analyzing the Avg_Open_To_Buy¶

Analyzing the Total_Amt_Chng_Q4_Q1¶

Analyzing the Total_Ct_Chng_Q4_Q1¶

Analyzing the Total_Trans_Amt¶

Analyzing the Total_Trans_Ct¶

Analyzing the Avg_Utilization_Ratio¶

Data Description Post Treament¶

Bivariate Analysis¶

Visualise variables association with Attrition parameter & its correlation¶

Analyzing the Categorial attributes with Attrition Flag¶

Analyzing the Numerical attributes with Product Taken¶

Multivariate Analysis - Visualise association with Product Taken & correlation between other Features¶

Model Building¶

Click to return to TOC¶

Data Preparation for Modeling¶

Split Data¶

Missing Value Treatment¶

Building the model¶

Click to return to TOC¶

Model evaluation criterion:¶

The model can make wrong predictions as:¶

Which case is more important?¶

How to reduce this loss i.e need to reduce False Negatives?¶

Model Analysis - Original data¶

Click to return to TOC¶

Oversampling training data using SMOTE¶

Model Analysis - Oversampling data¶

Click to return to TOC¶

Undersampling training data using Random Undersampler¶

Model Analysis - Undersampling data¶

Click to return to TOC¶

Comparison Models with - Original vs Oversample vs Undersample Data - 21 models¶

Adaboost¶

Gradient Boost¶

XGBoost¶

Comparing all models¶

Model Performance on Test dataset¶

Comparing the Training vs Testing data¶

Based on the earlier inference, we will consider AdaBoost Random as the best model without overfitting & good accuracy when compared with the other models and will use it for the further analysis¶

Observations¶

Pipelines for productionizing the model¶

Insights from Numerical Data ¶